src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java

// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.packages;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.LabelValidator;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.events.Location;
import com.google.devtools.build.lib.events.NullEventHandler;
import com.google.devtools.build.lib.events.StoredEventHandler;
import com.google.devtools.build.lib.packages.Globber.BadGlobException;
import com.google.devtools.build.lib.packages.License.DistributionType;
import com.google.devtools.build.lib.packages.Preprocessor.AstAfterPreprocessing;
import com.google.devtools.build.lib.packages.RuleFactory.BuildLangTypedAttributeValuesMap;
import com.google.devtools.build.lib.skylarkinterface.Param;
import com.google.devtools.build.lib.skylarkinterface.SkylarkSignature;
import com.google.devtools.build.lib.skylarkinterface.SkylarkValue;
import com.google.devtools.build.lib.syntax.AssignmentStatement;
import com.google.devtools.build.lib.syntax.BaseFunction;
import com.google.devtools.build.lib.syntax.BuildFileAST;
import com.google.devtools.build.lib.syntax.BuiltinFunction;
import com.google.devtools.build.lib.syntax.ClassObject;
import com.google.devtools.build.lib.syntax.Environment;
import com.google.devtools.build.lib.syntax.Environment.Extension;
import com.google.devtools.build.lib.syntax.Environment.Phase;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
import com.google.devtools.build.lib.syntax.Expression;
import com.google.devtools.build.lib.syntax.FuncallExpression;
import com.google.devtools.build.lib.syntax.FunctionSignature;
import com.google.devtools.build.lib.syntax.GlobList;
import com.google.devtools.build.lib.syntax.Identifier;
import com.google.devtools.build.lib.syntax.Mutability;
import com.google.devtools.build.lib.syntax.ParserInputSource;
import com.google.devtools.build.lib.syntax.Runtime;
import com.google.devtools.build.lib.syntax.SkylarkDict;
import com.google.devtools.build.lib.syntax.SkylarkList;
import com.google.devtools.build.lib.syntax.SkylarkList.MutableList;
import com.google.devtools.build.lib.syntax.SkylarkSignatureProcessor;
import com.google.devtools.build.lib.syntax.Statement;
import com.google.devtools.build.lib.syntax.Type;
import com.google.devtools.build.lib.syntax.Type.ConversionException;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.UnixGlob;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Logger;
import javax.annotation.Nullable;

/**
 * The package factory is responsible for constructing Package instances
 * from a BUILD file's abstract syntax tree (AST).
 *
 * <p>A PackageFactory is a heavy-weight object; create them sparingly.
 * Typically only one is needed per client application.
 */
public final class PackageFactory {
  /**
   * An argument to the {@code package()} function.
   */
  public abstract static class PackageArgument<T> {
    private final String name;
    private final Type<T> type;

    protected PackageArgument(String name, Type<T> type) {
      this.name = name;
      this.type = type;
    }

    public String getName() {
      return name;
    }

    private void convertAndProcess(
        Package.Builder pkgBuilder, Location location, Object value)
        throws EvalException {
      T typedValue = type.convert(value, "'package' argument", pkgBuilder.getBuildFileLabel());
      process(pkgBuilder, location, typedValue);
    }

    /**
     * Process an argument.
     *
     * @param pkgBuilder the package builder to be mutated
     * @param location the location of the {@code package} function for error reporting
     * @param value the value of the argument. Typically passed to {@link Type#convert}
     */
    protected abstract void process(
        Package.Builder pkgBuilder, Location location, T value)
        throws EvalException;
  }

  /**
   * An extension to the global namespace of the BUILD language.
   */
  // TODO(bazel-team): this is largely unrelated to syntax.Environment.Extension,
  // and should probably be renamed PackageFactory.RuntimeExtension, since really,
  // we're extending the Runtime with more classes.
  public interface EnvironmentExtension {
    /**
     * Update the global environment with the identifiers this extension contributes.
     */
    void update(Environment environment);

    /**
     * Update the global environment of WORKSPACE files.
     */
    void updateWorkspace(Environment environment);

    /**
     * Returns the extra functions needed to be added to the Skylark native module.
     */
    ImmutableList<BaseFunction> nativeModuleFunctions();

    /**
     * Returns the extra arguments to the {@code package()} statement.
     */
    Iterable<PackageArgument<?>> getPackageArguments();
  }

  private static class DefaultVisibility extends PackageArgument<List<Label>> {
    private DefaultVisibility() {
      super("default_visibility", BuildType.LABEL_LIST);
    }

    @Override
    protected void process(Package.Builder pkgBuilder, Location location,
        List<Label> value) throws EvalException{
      try {
        pkgBuilder.setDefaultVisibility(getVisibility(pkgBuilder.getBuildFileLabel(), value));
      } catch (EvalException e) {
        throw new EvalException(location, e.getMessage());
      }
    }
  }

  private static class DefaultTestOnly extends PackageArgument<Boolean> {
    private DefaultTestOnly() {
      super("default_testonly", Type.BOOLEAN);
    }

    @Override
    protected void process(Package.Builder pkgBuilder, Location location,
        Boolean value) {
      pkgBuilder.setDefaultTestonly(value);
    }
  }

  private static class DefaultDeprecation extends PackageArgument<String> {
    private DefaultDeprecation() {
      super("default_deprecation", Type.STRING);
    }

    @Override
    protected void process(Package.Builder pkgBuilder, Location location,
        String value) {
      pkgBuilder.setDefaultDeprecation(value);
    }
  }

  private static class Features extends PackageArgument<List<String>> {
    private Features() {
      super("features", Type.STRING_LIST);
    }

    @Override
    protected void process(Package.Builder pkgBuilder, Location location,
        List<String> value) {
      pkgBuilder.addFeatures(value);
    }
  }

  private static class DefaultLicenses extends PackageArgument<License> {
    private DefaultLicenses() {
      super("licenses", BuildType.LICENSE);
    }

    @Override
    protected void process(Package.Builder pkgBuilder, Location location,
        License value) {
      pkgBuilder.setDefaultLicense(value);
    }
  }

  private static class DefaultDistribs extends PackageArgument<Set<DistributionType>> {
    private DefaultDistribs() {
      super("distribs", BuildType.DISTRIBUTIONS);
    }

    @Override
    protected void process(Package.Builder pkgBuilder, Location location,
        Set<DistributionType> value) {
      pkgBuilder.setDefaultDistribs(value);
    }
  }

  /**
   * Declares the package() attribute specifying the default value for
   * {@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR} when not explicitly specified.
   */
  private static class DefaultCompatibleWith extends PackageArgument<List<Label>> {
    private DefaultCompatibleWith() {
      super(Package.DEFAULT_COMPATIBLE_WITH_ATTRIBUTE, BuildType.LABEL_LIST);
    }

    @Override
    protected void process(Package.Builder pkgBuilder, Location location,
        List<Label> value) {
      pkgBuilder.setDefaultCompatibleWith(value, Package.DEFAULT_COMPATIBLE_WITH_ATTRIBUTE,
          location);
    }
  }

  /**
   * Declares the package() attribute specifying the default value for
   * {@link RuleClass#RESTRICTED_ENVIRONMENT_ATTR} when not explicitly specified.
   */
  private static class DefaultRestrictedTo extends PackageArgument<List<Label>> {
    private DefaultRestrictedTo() {
      super(Package.DEFAULT_RESTRICTED_TO_ATTRIBUTE, BuildType.LABEL_LIST);
    }

    @Override
    protected void process(Package.Builder pkgBuilder, Location location,
        List<Label> value) {
      pkgBuilder.setDefaultRestrictedTo(value, Package.DEFAULT_RESTRICTED_TO_ATTRIBUTE, location);
    }
  }

  public static final String PKG_CONTEXT = "$pkg_context";

  // Used outside of Bazel!
  /** {@link Globber} that uses the legacy GlobCache. */
  public static class LegacyGlobber implements Globber {
    private final GlobCache globCache;
    private final boolean sort;

    private LegacyGlobber(GlobCache globCache, boolean sort) {
      this.globCache = globCache;
      this.sort = sort;
    }

    private static class Token extends Globber.Token {
      public final List<String> includes;
      public final List<String> excludes;
      public final boolean excludeDirs;

      public Token(List<String> includes, List<String> excludes, boolean excludeDirs) {
        this.includes = includes;
        this.excludes = excludes;
        this.excludeDirs = excludeDirs;
      }
    }

    @Override
    public Token runAsync(List<String> includes, List<String> excludes, boolean excludeDirs)
        throws BadGlobException {
      for (String pattern : Iterables.concat(includes, excludes)) {
        globCache.getGlobUnsortedAsync(pattern, excludeDirs);
      }
      return new Token(includes, excludes, excludeDirs);
    }

    @Override
    public List<String> fetch(Globber.Token token) throws IOException, InterruptedException {
      List<String> result;
      Token legacyToken = (Token) token;
      try {
        result = globCache.globUnsorted(legacyToken.includes, legacyToken.excludes,
            legacyToken.excludeDirs);
      } catch (BadGlobException e) {
        throw new IllegalStateException(e);
      }
      if (sort) {
        Collections.sort(result);
      }
      return result;
    }

    @Override
    public void onInterrupt() {
      globCache.cancelBackgroundTasks();
    }

    @Override
    public void onCompletion() {
      globCache.finishBackgroundTasks();
    }
  }

  private static final Logger LOG = Logger.getLogger(PackageFactory.class.getName());

  private final RuleFactory ruleFactory;
  private final RuleClassProvider ruleClassProvider;

  private AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls;
  private Preprocessor.Factory preprocessorFactory = Preprocessor.Factory.NullFactory.INSTANCE;

  private final ThreadPoolExecutor threadPool;
  private Map<String, String> platformSetRegexps;

  private final ImmutableList<EnvironmentExtension> environmentExtensions;
  private final ImmutableMap<String, PackageArgument<?>> packageArguments;

  private final Package.Builder.Helper packageBuilderHelper;

  /** Factory for {@link PackageFactory} instances. Intended to only be used by unit tests. */
  @VisibleForTesting
  public abstract static class FactoryForTesting {
    public final PackageFactory create(RuleClassProvider ruleClassProvider, FileSystem fs) {
      return create(ruleClassProvider, null, ImmutableList.<EnvironmentExtension>of(), fs);
    }
    
    public final PackageFactory create(
        RuleClassProvider ruleClassProvider,
        EnvironmentExtension environmentExtension,
        FileSystem fs) {
      return create(ruleClassProvider, null, ImmutableList.of(environmentExtension), fs);
    }
  
    public final PackageFactory create(
        RuleClassProvider ruleClassProvider,
        Map<String, String> platformSetRegexps,
        Iterable<EnvironmentExtension> environmentExtensions,
        FileSystem fs) {
      return create(
          ruleClassProvider,
          platformSetRegexps,
          AttributeContainer.ATTRIBUTE_CONTAINER_FACTORY,
          environmentExtensions,
          "test",
          fs);
    }
      
    protected abstract PackageFactory create(
        RuleClassProvider ruleClassProvider,
        Map<String, String> platformSetRegexps,
        Function<RuleClass, AttributeContainer> attributeContainerFactory,
        Iterable<EnvironmentExtension> environmentExtensions,
        String version,
        FileSystem fs);
  }

  /**
   * Constructs a {@code PackageFactory} instance with a specific glob path translator
   * and rule factory.
   *
   * <p>Only intended to be called by BlazeRuntime or {@link FactoryForTesting#create}.
   *
   * <p>Do not call this constructor directly in tests; please use
   * TestConstants#PACKAGE_FACTORY_FACTORY_FOR_TESTING instead.
   */
  public PackageFactory(
      RuleClassProvider ruleClassProvider,
      Map<String, String> platformSetRegexps,
      Function<RuleClass, AttributeContainer> attributeContainerFactory,
      Iterable<EnvironmentExtension> environmentExtensions,
      String version,
      Package.Builder.Helper packageBuilderHelper) {
    this.platformSetRegexps = platformSetRegexps;
    this.ruleFactory = new RuleFactory(ruleClassProvider, attributeContainerFactory);
    this.ruleClassProvider = ruleClassProvider;
    threadPool = new ThreadPoolExecutor(100, 100, 15L, TimeUnit.SECONDS,
        new LinkedBlockingQueue<Runnable>(),
        new ThreadFactoryBuilder().setNameFormat("Legacy globber %d").build());
    // Do not consume threads when not in use.
    threadPool.allowCoreThreadTimeOut(true);
    this.environmentExtensions = ImmutableList.copyOf(environmentExtensions);
    this.packageArguments = createPackageArguments();
    this.nativeModule = newNativeModule();
    this.workspaceNativeModule = WorkspaceFactory.newNativeModule(ruleClassProvider, version);
    this.packageBuilderHelper = packageBuilderHelper;
  }

  /**
   * Sets the preprocessor used.
   */
  public void setPreprocessorFactory(Preprocessor.Factory preprocessorFactory) {
    this.preprocessorFactory = preprocessorFactory;
  }

 /**
   * Sets the syscalls cache used in globbing.
   */
  public void setSyscalls(AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls) {
    this.syscalls = Preconditions.checkNotNull(syscalls);
  }

  /**
   * Sets the max number of threads to use for globbing.
   */
  public void setGlobbingThreads(int globbingThreads) {
    threadPool.setCorePoolSize(globbingThreads);
    threadPool.setMaximumPoolSize(globbingThreads);
  }


  /**
   * Returns the immutable, unordered set of names of all the known rule
   * classes.
   */
  public Set<String> getRuleClassNames() {
    return ruleFactory.getRuleClassNames();
  }

  /**
   * Returns the {@link RuleClass} for the specified rule class name.
   */
  public RuleClass getRuleClass(String ruleClassName) {
    return ruleFactory.getRuleClass(ruleClassName);
  }

  /**
   * Returns the {@link RuleClassProvider} of this {@link PackageFactory}.
   */
  public RuleClassProvider getRuleClassProvider() {
    return ruleClassProvider;
  }

  public ImmutableList<EnvironmentExtension> getEnvironmentExtensions() {
    return environmentExtensions;
  }

  /**
   * Creates the list of arguments for the 'package' function.
   */
  private ImmutableMap<String, PackageArgument<?>> createPackageArguments() {
    ImmutableList.Builder<PackageArgument<?>> arguments =
        ImmutableList.<PackageArgument<?>>builder()
            .add(new DefaultDeprecation())
            .add(new DefaultDistribs())
            .add(new DefaultLicenses())
            .add(new DefaultTestOnly())
            .add(new DefaultVisibility())
            .add(new Features())
            .add(new DefaultCompatibleWith())
            .add(new DefaultRestrictedTo());

    for (EnvironmentExtension extension : environmentExtensions) {
      arguments.addAll(extension.getPackageArguments());
    }

    ImmutableMap.Builder<String, PackageArgument<?>> packageArguments = ImmutableMap.builder();
    for (PackageArgument<?> argument : arguments.build()) {
      packageArguments.put(argument.getName(), argument);
    }
    return packageArguments.build();
  }

  /**
   * ************************************************************************** Environment function
   * factories.
   */

  /**
   * Returns a function-value implementing "glob" in the specified package context.
   *
   * @param async if true, start globs in the background but don't block on their completion. Only
   *     use this for heuristic preloading.
   */
  @SkylarkSignature(
    name = "glob",
    objectType = Object.class,
    returnType = SkylarkList.class,
    doc = "Returns a list of files that match glob search pattern",
    parameters = {
      @Param(
        name = "include",
        type = SkylarkList.class,
        generic1 = String.class,
        doc = "a list of strings specifying patterns of files to include."
      ),
      @Param(
        name = "exclude",
        type = SkylarkList.class,
        generic1 = String.class,
        defaultValue = "[]",
        positional = false,
        named = true,
        doc = "a list of strings specifying patterns of files to exclude."
      ),
      // TODO(bazel-team): migrate all existing code to use boolean instead?
      @Param(
        name = "exclude_directories",
        type = Integer.class,
        defaultValue = "1",
        positional = false,
        named = true,
        doc = "a integer that if non-zero indicates directories should not be matched."
      )
    },
    documented = false,
    useAst = true,
    useEnvironment = true
  )
  private static final BuiltinFunction.Factory newGlobFunction =
      new BuiltinFunction.Factory("glob") {
        public BuiltinFunction create(final PackageContext originalContext, final boolean async) {
          return new BuiltinFunction("glob", this) {
            public SkylarkList invoke(
                SkylarkList include,
                SkylarkList exclude,
                Integer excludeDirectories,
                FuncallExpression ast,
                Environment env)
                throws EvalException, ConversionException, InterruptedException {
              return callGlob(
                  originalContext, async, include, exclude, excludeDirectories != 0, ast, env);
            }
          };
        }
      };

  static SkylarkList callGlob(@Nullable PackageContext originalContext,
      boolean async, Object include, Object exclude, boolean excludeDirs,
      FuncallExpression ast, Environment env)
      throws EvalException, ConversionException, InterruptedException {
    // Skylark build extensions need to get the PackageContext from the Environment;
    // async glob functions cannot do the same because the Environment is not thread safe.
    PackageContext context;
    if (originalContext == null) {
      Preconditions.checkArgument(!async);
      context = getContext(env, ast);
    } else {
      context = originalContext;
    }

    List<String> includes = Type.STRING_LIST.convert(include, "'glob' argument");
    List<String> excludes = Type.STRING_LIST.convert(exclude, "'glob' argument");

    GlobList<String> globList;
    if (async) {
      try {
        context.globber.runAsync(includes, excludes, excludeDirs);
      } catch (BadGlobException e) {
        // Ignore: errors will appear during the actual evaluation of the package.
      }
      globList = GlobList.captureResults(includes, excludes, ImmutableList.<String>of());
    } else {
      globList = handleGlob(includes, excludes, excludeDirs, context, ast);
    }
    return new MutableList(globList, env);
  }

  /**
   * Adds a glob to the package, reporting any errors it finds.
   *
   * @param includes the list of includes which must be non-null
   * @param excludes the list of excludes which must be non-null
   * @param context the package context
   * @param ast the AST
   * @return the list of matches
   * @throws EvalException if globbing failed
   */
  private static GlobList<String> handleGlob(List<String> includes, List<String> excludes,
      boolean excludeDirs, PackageContext context, FuncallExpression ast)
        throws EvalException, InterruptedException {
    try {
      Globber.Token globToken = context.globber.runAsync(includes, excludes, excludeDirs);
      List<String> matches = context.globber.fetch(globToken);
      return GlobList.captureResults(includes, excludes, matches);
    } catch (IOException expected) {
      context.eventHandler.handle(Event.error(ast.getLocation(),
              "error globbing [" + Joiner.on(", ").join(includes) + "]: " + expected.getMessage()));
      context.pkgBuilder.setContainsErrors();
      return GlobList.captureResults(includes, excludes, ImmutableList.<String>of());
    } catch (BadGlobException e) {
      throw new EvalException(ast.getLocation(), e.getMessage());
    }
  }

  /**
   * Returns a function value implementing the "mocksubinclude" function,
   * emitted by the PythonPreprocessor.  We annotate the
   * package with additional dependencies.  (A 'real' subinclude will never be
   * seen by the parser, because the presence of "subinclude" triggers
   * preprocessing.)
   */
  @SkylarkSignature(name = "mocksubinclude", returnType = Runtime.NoneType.class,
      doc = "implement the mocksubinclude function emitted by the PythonPreprocessor",
      parameters = {
        @Param(name = "label", type = Object.class,
            doc = "a label designator."),
        @Param(name = "path", type = String.class,
            doc = "a path.")},
      documented = false, useLocation = true)
  private static final BuiltinFunction.Factory newMockSubincludeFunction =
      new BuiltinFunction.Factory("mocksubinclude") {
        public BuiltinFunction create(final PackageContext context) {
          return new BuiltinFunction("mocksubinclude", this) {
            public Runtime.NoneType invoke(Object labelO, String pathString,
                Location loc) throws ConversionException {
              Label label = BuildType.LABEL.convert(labelO, "'mocksubinclude' argument",
                  context.pkgBuilder.getBuildFileLabel());
              Path path = pathString.isEmpty()
                  ? null : context.pkgBuilder.getFilename().getRelative(pathString);
              // A subinclude within a package counts as a file declaration.
              if (label.getPackageIdentifier().equals(context.pkgBuilder.getPackageIdentifier())) {
                if (loc == null) {
                  loc = Location.fromFile(context.pkgBuilder.getFilename());
                }
                context.pkgBuilder.createInputFileMaybe(label, loc);
              }

              context.pkgBuilder.addSubinclude(label, path);
              return Runtime.NONE;
            }
          };
        }
      };

  /**
   * Returns a function value implementing "environment_group" in the specified package context.
   * Syntax is as follows:
   *
   * <pre>{@code
   *   environment_group(
   *       name = "sample_group",
   *       environments = [":env1", ":env2", ...],
   *       defaults = [":env1", ...]
   *   )
   * }</pre>
   *
   * <p>Where ":env1", "env2", ... are all environment rules declared in the same package. All
   * parameters are mandatory.
   */
  @SkylarkSignature(name = "environment_group", returnType = Runtime.NoneType.class,
      doc = "Defines a set of related environments that can be tagged onto rules to prevent"
      + "incompatible rules from depending on each other.",
      parameters = {
        @Param(name = "name", type = String.class, positional = false, named = true,
            doc = "The name of the rule."),
        // Both parameter below are lists of label designators
        @Param(name = "environments", type = SkylarkList.class, generic1 = Object.class,
            positional = false, named = true,
            doc = "A list of Labels for the environments to be grouped, from the same package."),
        @Param(name = "defaults", type = SkylarkList.class, generic1 = Object.class,
            positional = false, named = true,
            doc = "A list of Labels.")}, // TODO(bazel-team): document what that is
      documented = false, useLocation = true)
  private static final BuiltinFunction.Factory newEnvironmentGroupFunction =
      new BuiltinFunction.Factory("environment_group") {
        public BuiltinFunction create(final PackageContext context) {
          return new BuiltinFunction("environment_group", this) {
            public Runtime.NoneType invoke(
                String name, SkylarkList environmentsList, SkylarkList defaultsList,
                Location loc) throws EvalException, ConversionException {
              List<Label> environments = BuildType.LABEL_LIST.convert(environmentsList,
                  "'environment_group argument'", context.pkgBuilder.getBuildFileLabel());
              List<Label> defaults = BuildType.LABEL_LIST.convert(defaultsList,
                  "'environment_group argument'", context.pkgBuilder.getBuildFileLabel());

              try {
                context.pkgBuilder.addEnvironmentGroup(
                    name, environments, defaults, context.eventHandler, loc);
                return Runtime.NONE;
              } catch (LabelSyntaxException e) {
                throw new EvalException(loc,
                    "environment group has invalid name: " + name + ": " + e.getMessage());
              } catch (Package.NameConflictException e) {
                throw new EvalException(loc, e.getMessage());
              }
            }
          };
        }
      };

  /**
   * Returns a function-value implementing "exports_files" in the specified
   * package context.
   */
  @SkylarkSignature(name = "exports_files", returnType = Runtime.NoneType.class,
      doc = "Declare a set of files as exported",
      parameters = {
        @Param(name = "srcs", type = SkylarkList.class, generic1 = String.class,
            doc = "A list of strings, the names of the files to export."),
        // TODO(blaze-team): make it possible to express a list of label designators,
        // i.e. a java List or Skylark list of Label or String.
        @Param(name = "visibility", type = SkylarkList.class, noneable = true,
            defaultValue = "None",
            doc = "A list of Labels specifying the visibility of the exported files "
            + "(defaults to public)"),
        @Param(name = "licenses", type = SkylarkList.class, generic1 = String.class,
            noneable = true, defaultValue = "None",
            doc = "A list of strings specifying the licenses used in the exported code.")},
      documented = false, useAst = true, useEnvironment = true)
  private static final BuiltinFunction.Factory newExportsFilesFunction =
      new BuiltinFunction.Factory("exports_files") {
        public BuiltinFunction create () {
          return new BuiltinFunction("exports_files", this) {
            public Runtime.NoneType invoke(
                SkylarkList srcs, Object visibility, Object licenses,
                FuncallExpression ast, Environment env)
                throws EvalException, ConversionException {
              return callExportsFiles(srcs, visibility, licenses, ast, env);
            }
          };
        }
      };

  static Runtime.NoneType callExportsFiles(Object srcs, Object visibilityO, Object licensesO,
      FuncallExpression ast, Environment env) throws EvalException, ConversionException {
    Package.Builder pkgBuilder = getContext(env, ast).pkgBuilder;
    List<String> files = Type.STRING_LIST.convert(srcs, "'exports_files' operand");

    RuleVisibility visibility;
    try {
      visibility = EvalUtils.isNullOrNone(visibilityO)
          ? ConstantRuleVisibility.PUBLIC
          : getVisibility(pkgBuilder.getBuildFileLabel(), BuildType.LABEL_LIST.convert(
              visibilityO,
              "'exports_files' operand",
              pkgBuilder.getBuildFileLabel()));
    } catch (EvalException e) {
      throw new EvalException(ast.getLocation(), e.getMessage());
    }
    // TODO(bazel-team): is licenses plural or singular?
    License license = BuildType.LICENSE.convertOptional(licensesO, "'exports_files' operand");

    for (String file : files) {
      String errorMessage = LabelValidator.validateTargetName(file);
      if (errorMessage != null) {
        throw new EvalException(ast.getLocation(), errorMessage);
      }
      try {
        InputFile inputFile = pkgBuilder.createInputFile(file, ast.getLocation());
        if (inputFile.isVisibilitySpecified()
            && inputFile.getVisibility() != visibility) {
          throw new EvalException(ast.getLocation(),
              String.format("visibility for exported file '%s' declared twice",
                  inputFile.getName()));
        }
        if (license != null && inputFile.isLicenseSpecified()) {
          throw new EvalException(ast.getLocation(),
              String.format("licenses for exported file '%s' declared twice",
                  inputFile.getName()));
        }
        if (license == null && pkgBuilder.getDefaultLicense() == License.NO_LICENSE
            && RuleClass.isThirdPartyPackage(pkgBuilder.getPackageIdentifier())) {
          throw new EvalException(ast.getLocation(),
              "third-party file '" + inputFile.getName() + "' lacks a license declaration "
              + "with one of the following types: notice, reciprocal, permissive, "
              + "restricted, unencumbered, by_exception_only");
        }

        pkgBuilder.setVisibilityAndLicense(inputFile, visibility, license);
      } catch (Package.Builder.GeneratedLabelConflict e) {
        throw new EvalException(ast.getLocation(), e.getMessage());
      }
    }
    return Runtime.NONE;
  }

  /**
   * Returns a function-value implementing "licenses" in the specified package
   * context.
   * TODO(bazel-team): Remove in favor of package.licenses.
   */
  @SkylarkSignature(name = "licenses", returnType = Runtime.NoneType.class,
      doc = "Declare the license(s) for the code in the current package.",
      parameters = {
        @Param(name = "license_strings", type = SkylarkList.class, generic1 = String.class,
            doc = "A list of strings, the names of the licenses used.")},
      documented = false, useLocation = true)
  private static final BuiltinFunction.Factory newLicensesFunction =
      new BuiltinFunction.Factory("licenses") {
        public BuiltinFunction create(final PackageContext context) {
          return new BuiltinFunction("licenses", this) {
            public Runtime.NoneType invoke(SkylarkList licensesList, Location loc) {
              try {
                License license = BuildType.LICENSE.convert(licensesList, "'licenses' operand");
                context.pkgBuilder.setDefaultLicense(license);
              } catch (ConversionException e) {
                context.eventHandler.handle(Event.error(loc, e.getMessage()));
                context.pkgBuilder.setContainsErrors();
              }
              return Runtime.NONE;
            }
          };
        }
      };

  /**
   * Returns a function-value implementing "distribs" in the specified package
   * context.
   */
  // TODO(bazel-team): Remove in favor of package.distribs.
  // TODO(bazel-team): Remove all these new*Function-s and/or have static functions
  // that consult the context dynamically via getContext(env, ast) since we have that,
  // and share the functions with the native package... which requires unifying the List types.
  @SkylarkSignature(name = "distribs", returnType = Runtime.NoneType.class,
      doc = "Declare the distribution(s) for the code in the current package.",
      parameters = {
        @Param(name = "distribution_strings", type = Object.class,
            doc = "The distributions.")},
      documented = false, useLocation = true)
  private static final BuiltinFunction.Factory newDistribsFunction =
      new BuiltinFunction.Factory("distribs") {
        public BuiltinFunction create(final PackageContext context) {
          return new BuiltinFunction("distribs", this) {
            public Runtime.NoneType invoke(Object object, Location loc) {
              try {
                Set<DistributionType> distribs = BuildType.DISTRIBUTIONS.convert(object,
                    "'distribs' operand");
                context.pkgBuilder.setDefaultDistribs(distribs);
              } catch (ConversionException e) {
                context.eventHandler.handle(Event.error(loc, e.getMessage()));
                context.pkgBuilder.setContainsErrors();
              }
              return Runtime.NONE;
            }
          };
        }
      };

  @SkylarkSignature(name = "package_group", returnType = Runtime.NoneType.class,
      doc = "Declare a set of files as exported",
      parameters = {
        @Param(name = "name", type = String.class, named = true, positional = false,
            doc = "The name of the rule."),
        @Param(name = "packages", type = SkylarkList.class, generic1 = String.class,
            defaultValue = "[]",
            named = true,
            positional = false,
            doc = "A list of Strings specifying the packages grouped."),
        // java list or list of label designators: Label or String
        @Param(name = "includes", type = SkylarkList.class, generic1 = Object.class,
            defaultValue = "[]",
            named = true,
            positional = false,
            doc = "A list of Label specifiers for the files to include.")},
      documented = false, useAst = true, useEnvironment = true)
  private static final BuiltinFunction.Factory newPackageGroupFunction =
      new BuiltinFunction.Factory("package_group") {
        public BuiltinFunction create() {
          return new BuiltinFunction("package_group", this) {
            public Runtime.NoneType invoke(String name, SkylarkList packages, SkylarkList includes,
                FuncallExpression ast, Environment env) throws EvalException, ConversionException {
              return callPackageFunction(name, packages, includes, ast, env);
            }
          };
        }
      };

  @Nullable
  static SkylarkDict<String, Object> callGetRuleFunction(
      String name, FuncallExpression ast, Environment env)
      throws EvalException, ConversionException {
    PackageContext context = getContext(env, ast);
    Target target = context.pkgBuilder.getTarget(name);

    return targetDict(target, ast.getLocation(), env);
  }

  @Nullable
  private static SkylarkDict<String, Object> targetDict(
      Target target, Location loc, Environment env)
      throws NotRepresentableException, EvalException {
    if (target == null || !(target instanceof Rule)) {
      return null;
    }
    SkylarkDict<String, Object> values = SkylarkDict.<String, Object>of(env);

    Rule rule = (Rule) target;
    AttributeContainer cont = rule.getAttributeContainer();
    for (Attribute attr : rule.getAttributes()) {
      if (!Character.isAlphabetic(attr.getName().charAt(0))) {
        continue;
      }

      if (attr.getName().equals("distribs")) {
        // attribute distribs: cannot represent type class java.util.Collections$SingletonSet
        // in Skylark: [INTERNAL].
        continue;
      }

      try {
        Object val = skylarkifyValue(cont.getAttr(attr.getName()), target.getPackage());
        if (val == null) {
          continue;
        }
        values.put(attr.getName(), val, loc, env);
      } catch (NotRepresentableException e) {
        throw new NotRepresentableException(
            String.format(
                "target %s, attribute %s: %s", target.getName(), attr.getName(), e.getMessage()));
      }
    }

    values.put("name", rule.getName(), loc, env);
    values.put("kind", rule.getRuleClass(), loc, env);
    return values;
  }

  static class NotRepresentableException extends EvalException {
    NotRepresentableException(String msg) {
      super(null, msg);
    }
  };

  /**
   * Converts back to type that will work in BUILD and skylark,
   * such as string instead of label, SkylarkList instead of List,
   * Returns null if we don't want to export the value.
   *
   * <p>All of the types returned are immutable. If we want, we can change this to
   * immutable in the future, but this is the safe choice for now.
   */
  @Nullable
  private static Object skylarkifyValue(Object val, Package pkg) throws NotRepresentableException {
    // TODO(bazel-team): the location of this function is ad-hoc. Arguably, the conversion
    // from Java native types to Skylark types should be part of the Type class hierarchy,
    if (val == null) {
      return null;
    }
    if (val instanceof Boolean) {
      return val;
    }
    if (val instanceof Integer) {
      return val;
    }
    if (val instanceof String) {
      return val;
    }

    if (val instanceof TriState) {
      switch ((TriState) val) {
        case AUTO:
          return Integer.valueOf(-1);
        case YES:
          return Integer.valueOf(1);
        case NO:
          return Integer.valueOf(0);
      }
    }

    if (val instanceof Label) {
      Label l = (Label) val;
      if (l.getPackageName().equals(pkg.getName())) {
        return ":" + l.getName();
      }
      return l.getCanonicalForm();
    }

    if (val instanceof List) {
      List<Object> l = new ArrayList<>();
      for (Object o : (List) val) {
        Object elt = skylarkifyValue(o, pkg);
        if (elt == null) {
          continue;
        }

        l.add(elt);
      }

      return SkylarkList.Tuple.copyOf(l);
    }
    if (val instanceof Map) {
      Map<Object, Object> m = new TreeMap<>();
      for (Map.Entry<?, ?> e : ((Map<?, ?>) val).entrySet()) {
        Object key = skylarkifyValue(e.getKey(), pkg);
        Object mapVal = skylarkifyValue(e.getValue(), pkg);

        if (key == null || mapVal == null) {
          continue;
        }

        m.put(key, mapVal);
      }
      return m;
    }
    if (val.getClass().isAnonymousClass()) {
      // Computed defaults. They will be represented as
      // "deprecation": com.google.devtools.build.lib.analysis.BaseRuleClasses$2@6960884a,
      // Filter them until we invent something more clever.
      return null;
    }

    if (val instanceof SkylarkValue) {
      return val;
    }

    if (val instanceof License) {
      // TODO(bazel-team): convert License.getLicenseTypes() to a list of strings.
      return null;
    }

    if (val instanceof BuildType.SelectorList) {
      // This is terrible:
      //  1) this value is opaque, and not a BUILD value, so it cannot be used in rule arguments
      //  2) its representation has a pointer address, so it breaks hermeticity.
      //
      // Even though this is clearly imperfect, we return this value because otherwise
      // native.rules() fails if there is any rule using a select() in the BUILD file.
      //
      // To remedy this, we should return a syntax.SelectorList. To do so, we have to
      // 1) recurse into the Selector contents of SelectorList, so those values are skylarkified too
      // 2) get the right Class<?> value. We could probably get at that by looking at
      //    ((SelectorList)val).getSelectors().first().getEntries().first().getClass().

      return val;
    }

    // We are explicit about types we don't understand so we minimize changes to existing callers
    // if we add more types that we can represent.
    throw new NotRepresentableException(
        String.format(
            "cannot represent %s (%s) in skylark", val.toString(), val.getClass().toString()));
  }


  static SkylarkDict<String, SkylarkDict<String, Object>> callGetRulesFunction(
      FuncallExpression ast, Environment env)
      throws EvalException {
    PackageContext context = getContext(env, ast);
    Collection<Target> targets = context.pkgBuilder.getTargets();
    Location loc = ast.getLocation();

    // Sort by name.
    SkylarkDict<String, SkylarkDict<String, Object>> rules =
        SkylarkDict.<String, SkylarkDict<String, Object>>of(env);
    for (Target t : targets) {
      if (t instanceof Rule) {
        SkylarkDict<String, Object> m = targetDict(t, loc, env);
        Preconditions.checkNotNull(m);
        rules.put(t.getName(), m, loc, env);
      }
    }

    return rules;
  }

  static Runtime.NoneType callPackageFunction(String name, Object packagesO, Object includesO,
      FuncallExpression ast, Environment env) throws EvalException, ConversionException {
    PackageContext context = getContext(env, ast);

    List<String> packages = Type.STRING_LIST.convert(
        packagesO, "'package_group.packages argument'");
    List<Label> includes = BuildType.LABEL_LIST.convert(includesO,
        "'package_group.includes argument'", context.pkgBuilder.getBuildFileLabel());

    try {
      context.pkgBuilder.addPackageGroup(name, packages, includes, context.eventHandler,
          ast.getLocation());
      return Runtime.NONE;
    } catch (LabelSyntaxException e) {
      throw new EvalException(ast.getLocation(),
          "package group has invalid name: " + name + ": " + e.getMessage());
    } catch (Package.NameConflictException e) {
      throw new EvalException(ast.getLocation(), e.getMessage());
    }
  }

  public static RuleVisibility getVisibility(Label ruleLabel, List<Label> original)
      throws EvalException {
    RuleVisibility result;

    result = ConstantRuleVisibility.tryParse(original);
    if (result != null) {
      return result;
    }

    result = PackageGroupsRuleVisibility.tryParse(ruleLabel, original);
    return result;
  }

  /**
   * Returns a function-value implementing "package" in the specified package
   * context.
   */
  private static BaseFunction newPackageFunction(
      final ImmutableMap<String, PackageArgument<?>> packageArguments) {
    // Flatten the map of argument name of PackageArgument specifier in two co-indexed arrays:
    // one for the argument names, to create a FunctionSignature when we create the function,
    // one of the PackageArgument specifiers, over which to iterate at every function invocation
    // at the same time that we iterate over the function arguments.
    final int numArgs = packageArguments.size();
    final String[] argumentNames = new String[numArgs];
    final PackageArgument<?>[] argumentSpecifiers = new PackageArgument<?>[numArgs];
    int i = 0;
    for (Map.Entry<String, PackageArgument<?>> entry : packageArguments.entrySet()) {
      argumentNames[i] = entry.getKey();
      argumentSpecifiers[i++] = entry.getValue();
    }

    return new BaseFunction("package", FunctionSignature.namedOnly(0, argumentNames)) {
      @Override
      public Object call(Object[] arguments, FuncallExpression ast, Environment env)
          throws EvalException {

        Package.Builder pkgBuilder = getContext(env, ast).pkgBuilder;

        // Validate parameter list
        if (pkgBuilder.isPackageFunctionUsed()) {
          throw new EvalException(ast.getLocation(),
              "'package' can only be used once per BUILD file");
        }
        pkgBuilder.setPackageFunctionUsed();

        // Parse params
        boolean foundParameter = false;

        for (int i = 0; i < numArgs; i++) {
          Object value = arguments[i];
          if (value != null) {
            foundParameter = true;
            argumentSpecifiers[i].convertAndProcess(pkgBuilder, ast.getLocation(), value);
          }
        }

        if (!foundParameter) {
          throw new EvalException(ast.getLocation(),
              "at least one argument must be given to the 'package' function");
        }

        return Runtime.NONE;
      }
    };
  }

  // Helper function for createRuleFunction.
  private static void addRule(RuleFactory ruleFactory,
                              String ruleClassName,
                              PackageContext context,
                              Map<String, Object> kwargs,
                              FuncallExpression ast,
                              Environment env)
      throws RuleFactory.InvalidRuleException, Package.NameConflictException, InterruptedException {
    RuleClass ruleClass = getBuiltInRuleClass(ruleClassName, ruleFactory);
    BuildLangTypedAttributeValuesMap attributeValues = new BuildLangTypedAttributeValuesMap(kwargs);
    AttributeContainer attributeContainer = ruleFactory.getAttributeContainer(ruleClass);
    RuleFactory.createAndAddRule(context, ruleClass, attributeValues, ast, env, attributeContainer);
  }

  private static RuleClass getBuiltInRuleClass(String ruleClassName, RuleFactory ruleFactory) {
    if (ruleFactory.getRuleClassNames().contains(ruleClassName)) {
      return ruleFactory.getRuleClass(ruleClassName);
    }
    throw new IllegalArgumentException("no such rule class: "  + ruleClassName);
  }

  /**
   * Get the PackageContext by looking up in the environment.
   */
  public static PackageContext getContext(Environment env, FuncallExpression ast)
      throws EvalException {
    PackageContext value = (PackageContext) env.lookup(PKG_CONTEXT);
    if (value == null) {
      // if PKG_CONTEXT is missing, we're not called from a BUILD file. This happens if someone
      // uses native.some_func() in the wrong place.
      throw new EvalException(ast.getLocation(),
          "The native module cannot be accessed from here. "
          + "Wrap the function in a macro and call it from a BUILD file");
    }
    return value;
  }

  /**
   * Returns a function-value implementing the build rule "ruleClass" (e.g. cc_library) in the
   * specified package context.
   */
  private static BuiltinFunction newRuleFunction(
      final RuleFactory ruleFactory, final String ruleClass) {
    return new BuiltinFunction(ruleClass, FunctionSignature.KWARGS, BuiltinFunction.USE_AST_ENV) {
      @SuppressWarnings({"unchecked", "unused"})
      public Runtime.NoneType invoke(Map<String, Object> kwargs,
          FuncallExpression ast, Environment env)
          throws EvalException, InterruptedException {
        env.checkLoadingOrWorkspacePhase(ruleClass, ast.getLocation());
        try {
          addRule(ruleFactory, ruleClass, getContext(env, ast), kwargs, ast, env);
        } catch (RuleFactory.InvalidRuleException | Package.NameConflictException e) {
          throw new EvalException(ast.getLocation(), e.getMessage());
        }
        return Runtime.NONE;
      }
    };
  }

  /****************************************************************************
   * Package creation.
   */

  /**
   * Loads, scans parses and evaluates the build file at "buildFile", and
   * creates and returns a Package builder instance capable of building a package identified by
   * "packageId".
   *
   * <p>This method returns a builder to allow the caller to do additional work, if necessary.
   *
   * <p>This method assumes "packageId" is a valid package name according to the
   * {@link LabelValidator#validatePackageName} heuristic.
   *
   * <p>See {@link #evaluateBuildFile} for information on AST retention.
   *
   * <p>Executes {@code globber.onCompletion()} on completion and executes
   * {@code globber.onInterrupt()} on an {@link InterruptedException}.
   */
  // Used outside of bazel!
  public Package.Builder createPackageFromPreprocessingResult(
      Package externalPkg,
      PackageIdentifier packageId,
      Path buildFile,
      Preprocessor.Result preprocessingResult,
      List<Statement> preludeStatements,
      Map<String, Extension> imports,
      ImmutableList<Label> skylarkFileDependencies,
      RuleVisibility defaultVisibility,
      Globber globber) throws InterruptedException {
    StoredEventHandler localReporterForParsing = new StoredEventHandler();
    // Run the lexer and parser with a local reporter, so that errors from other threads do not
    // show up below.
    BuildFileAST buildFileAST = parseBuildFile(packageId, preprocessingResult.result,
        preludeStatements, localReporterForParsing);
    AstAfterPreprocessing astAfterPreprocessing = new AstAfterPreprocessing(preprocessingResult,
        buildFileAST, localReporterForParsing);
    return createPackageFromPreprocessingAst(
        externalPkg,
        packageId,
        buildFile,
        astAfterPreprocessing,
        imports,
        skylarkFileDependencies,
        defaultVisibility,
        globber);
  }

  public static BuildFileAST parseBuildFile(PackageIdentifier packageId, ParserInputSource in,
      List<Statement> preludeStatements, EventHandler eventHandler) {
    // Logged messages are used as a testability hook tracing the parsing progress
    LOG.fine("Starting to parse " + packageId);
    BuildFileAST buildFileAST = BuildFileAST.parseBuildFile(in, preludeStatements, eventHandler);
    LOG.fine("Finished parsing of " + packageId);
    return buildFileAST;
  }

  public Package.Builder createPackageFromPreprocessingAst(
      Package externalPkg,
      PackageIdentifier packageId,
      Path buildFile,
      Preprocessor.AstAfterPreprocessing astAfterPreprocessing,
      Map<String, Extension> imports,
      ImmutableList<Label> skylarkFileDependencies,
      RuleVisibility defaultVisibility,
      Globber globber) throws InterruptedException {
    MakeEnvironment.Builder makeEnv = new MakeEnvironment.Builder();
    if (platformSetRegexps != null) {
      makeEnv.setPlatformSetRegexps(platformSetRegexps);
    }
    try {
      // At this point the package is guaranteed to exist.  It may have parse or
      // evaluation errors, resulting in a diminished number of rules.
      prefetchGlobs(packageId, astAfterPreprocessing.ast, astAfterPreprocessing.preprocessed,
          buildFile, globber, defaultVisibility, makeEnv);
      return evaluateBuildFile(
          externalPkg,
          packageId,
          astAfterPreprocessing.ast,
          buildFile,
          globber,
          astAfterPreprocessing.allEvents,
          defaultVisibility,
          astAfterPreprocessing.containsPreprocessingErrors,
          makeEnv,
          imports,
          skylarkFileDependencies);
    } catch (InterruptedException e) {
      globber.onInterrupt();
      throw e;
    } finally {
      globber.onCompletion();
    }
  }

  @VisibleForTesting
  public Package.Builder newExternalPackageBuilder(Path workspacePath, String runfilesPrefix) {
    return Package.newExternalPackageBuilder(packageBuilderHelper, workspacePath, runfilesPrefix);
  }

  @VisibleForTesting
  public Package.Builder newPackageBuilder(PackageIdentifier packageId, String runfilesPrefix) {
    return new Package.Builder(packageBuilderHelper, packageId, runfilesPrefix);
  }

  @VisibleForTesting
  public Package createPackageForTesting(
      PackageIdentifier packageId,
      Path buildFile,
      CachingPackageLocator locator,
      EventHandler eventHandler)
      throws NoSuchPackageException, InterruptedException {
    Package externalPkg = newExternalPackageBuilder(
        buildFile.getRelative("WORKSPACE"), "TESTING").build();
    return createPackageForTesting(packageId, externalPkg, buildFile, locator, eventHandler);
  }

  /**
   * Same as createPackage, but does the required validation of "packageName" first,
   * throwing a {@link NoSuchPackageException} if the name is invalid.
   */
  @VisibleForTesting
  public Package createPackageForTesting(
      PackageIdentifier packageId,
      Package externalPkg,
      Path buildFile,
      CachingPackageLocator locator,
      EventHandler eventHandler)
      throws NoSuchPackageException, InterruptedException {
    String error =
        LabelValidator.validatePackageName(packageId.getPackageFragment().getPathString());
    if (error != null) {
      throw new BuildFileNotFoundException(
          packageId, "illegal package name: '" + packageId + "' (" + error + ")");
    }
    byte[] buildFileBytes = maybeGetBuildFileBytes(buildFile, eventHandler);
    if (buildFileBytes == null) {
      throw new BuildFileContainsErrorsException(packageId, "IOException occured");
    }

    Globber globber = createLegacyGlobber(buildFile.getParentDirectory(), packageId, locator);
    Preprocessor.Result preprocessingResult;
    try {
      preprocessingResult = preprocess(buildFile, packageId, buildFileBytes, globber);
    } catch (IOException e) {
      eventHandler.handle(
          Event.error(Location.fromFile(buildFile), "preprocessing failed: " + e.getMessage()));
      throw new BuildFileContainsErrorsException(packageId, "preprocessing failed", e);
    }

    Package result =
        createPackageFromPreprocessingResult(
                externalPkg,
                packageId,
                buildFile,
                preprocessingResult,
                /*preludeStatements=*/ImmutableList.<Statement>of(),
                /*imports=*/ImmutableMap.<String, Extension>of(),
                /*skylarkFileDependencies=*/ImmutableList.<Label>of(),
                /*defaultVisibility=*/ConstantRuleVisibility.PUBLIC,
                globber)
            .build();
    Event.replayEventsOn(eventHandler, result.getEvents());
    return result;
  }

  /** Preprocesses the given BUILD file. */
  public Preprocessor.Result preprocess(
      PackageIdentifier packageId, Path buildFile, CachingPackageLocator locator)
      throws InterruptedException, IOException {
    byte[] buildFileBytes =
        FileSystemUtils.readWithKnownFileSize(buildFile, buildFile.getFileSize());
    Globber globber = createLegacyGlobber(buildFile.getParentDirectory(), packageId, locator);
    try {
      return preprocess(buildFile, packageId, buildFileBytes, globber);
    } finally {
      globber.onCompletion();
    }
  }

  /**
   * Preprocesses the given BUILD file, executing {@code globber.onInterrupt()} on an
   * {@link InterruptedException}.
   */
  public Preprocessor.Result preprocess(
      Path buildFilePath, PackageIdentifier packageId, byte[] buildFileBytes,
      Globber globber) throws InterruptedException, IOException {
    Preprocessor preprocessor = preprocessorFactory.getPreprocessor();
    if (preprocessor == null) {
      return Preprocessor.Result.noPreprocessing(buildFilePath.asFragment(), buildFileBytes);
    }
    try {
      return preprocessor.preprocess(
          buildFilePath,
          buildFileBytes,
          packageId.toString(),
          globber,
          Environment.BUILD,
          ruleFactory.getRuleClassNames());
    } catch (InterruptedException e) {
      globber.onInterrupt();
      throw e;
    }
  }

  /** Returns a new {@link LegacyGlobber}. */
  public LegacyGlobber createLegacyGlobber(
      Path packageDirectory,
      PackageIdentifier packageId,
      CachingPackageLocator locator) {
    return createLegacyGlobber(new GlobCache(packageDirectory, packageId, locator, syscalls,
        threadPool));
  }

  /** Returns a new {@link LegacyGlobber}. */
  public static LegacyGlobber createLegacyGlobber(GlobCache globCache) {
    return new LegacyGlobber(globCache, /*sort=*/ true);
  }

  /**
   * Returns a new {@link LegacyGlobber}, the same as in {@link #createLegacyGlobber}, except that
   * the implementation of {@link Globber#fetch} intentionally breaks the contract and doesn't
   * return sorted results.
   */
  public LegacyGlobber createLegacyGlobberThatDoesntSort(
      Path packageDirectory,
      PackageIdentifier packageId,
      CachingPackageLocator locator) {
    return new LegacyGlobber(new GlobCache(packageDirectory, packageId, locator, syscalls,
        threadPool), /*sort=*/ false);
  }

  @Nullable
  private byte[] maybeGetBuildFileBytes(Path buildFile, EventHandler eventHandler) {
    try {
      return FileSystemUtils.readWithKnownFileSize(buildFile, buildFile.getFileSize());
    } catch (IOException e) {
      eventHandler.handle(Event.error(Location.fromFile(buildFile), e.getMessage()));
      return null;
    }
  }

  /**
   * This tuple holds the current package builder, current lexer, etc, for the
   * duration of the evaluation of one BUILD file. (We use a PackageContext
   * object in preference to storing these values in mutable fields of the
   * PackageFactory.)
   *
   * <p>PLEASE NOTE: references to PackageContext objects are held by many
   * BaseFunction closures, but should become unreachable once the Environment is
   * discarded at the end of evaluation.  Please be aware of your memory
   * footprint when making changes here!
   */
  public static class PackageContext {
    final Package.Builder pkgBuilder;
    final Globber globber;
    final EventHandler eventHandler;
    private final Function<RuleClass, AttributeContainer> attributeContainerFactory;

    @VisibleForTesting
    public PackageContext(
        Package.Builder pkgBuilder,
        Globber globber,
        EventHandler eventHandler,
        Function<RuleClass, AttributeContainer> attributeContainerFactory) {
      this.pkgBuilder = pkgBuilder;
      this.eventHandler = eventHandler;
      this.globber = globber;
      this.attributeContainerFactory = attributeContainerFactory;
    }

    /**
     * Returns the Label of this Package.
     */
    public Label getLabel() {
      return pkgBuilder.getBuildFileLabel();
    }

    /**
     * Returns the MakeEnvironment Builder of this Package.
     */
    public MakeEnvironment.Builder getMakeEnvironment() {
      return pkgBuilder.getMakeEnvironment();
    }

    /**
     * Returns the builder of this Package.
     */
    public Package.Builder getBuilder() {
      return pkgBuilder;
    }

    public Function<RuleClass, AttributeContainer> getAttributeContainerFactory() {
      return attributeContainerFactory;
    }
  }

  private final ClassObject nativeModule;
  private final ClassObject workspaceNativeModule;

  /** @return the Skylark struct to bind to "native" */
  public ClassObject getNativeModule(boolean workspace) {
    return workspace ? workspaceNativeModule : nativeModule;
  }

  /**
   * Returns a native module with the functions created using the {@link RuleClassProvider}
   * of this {@link PackageFactory}.
   */
  private ClassObject newNativeModule() {
    ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
    for (String nativeFunction : Runtime.getFunctionNames(SkylarkNativeModule.class)) {
      builder.put(nativeFunction, Runtime.getFunction(SkylarkNativeModule.class, nativeFunction));
    }
    for (String ruleClass : ruleFactory.getRuleClassNames()) {
      builder.put(ruleClass, newRuleFunction(ruleFactory, ruleClass));
    }
    builder.put("package", newPackageFunction(packageArguments));
    for (EnvironmentExtension extension : environmentExtensions) {
      for (BaseFunction function : extension.nativeModuleFunctions()) {
        builder.put(function.getName(), function);
      }
    }
    return SkylarkClassObjectConstructor.STRUCT.create(
        builder.build(), "no native function or rule '%s'");
  }

  private void buildPkgEnv(Environment pkgEnv, PackageContext context, RuleFactory ruleFactory) {
    // TODO(bazel-team): remove the naked functions that are redundant with the nativeModule,
    // or if not possible, at least make them straight copies from the native module variant.
    // or better, use a common Environment.Frame for these common bindings
    // (that shares a backing ImmutableMap for the bindings?)
    pkgEnv
        .setup("native", nativeModule)
        .setup("distribs", newDistribsFunction.apply(context))
        .setup("glob", newGlobFunction.apply(context, /*async=*/false))
        .setup("mocksubinclude", newMockSubincludeFunction.apply(context))
        .setup("licenses", newLicensesFunction.apply(context))
        .setup("exports_files", newExportsFilesFunction.apply())
        .setup("package_group", newPackageGroupFunction.apply())
        .setup("package", newPackageFunction(packageArguments))
        .setup("environment_group", newEnvironmentGroupFunction.apply(context));

    for (String ruleClass : ruleFactory.getRuleClassNames()) {
      BaseFunction ruleFunction = newRuleFunction(ruleFactory, ruleClass);
      pkgEnv.setup(ruleClass, ruleFunction);
    }

    for (EnvironmentExtension extension : environmentExtensions) {
      extension.update(pkgEnv);
    }
  }

  /**
   * Called by a caller of {@link #createPackageFromPreprocessingAst} after this caller has fully
   * loaded the package.
   */
  public void afterDoneLoadingPackage(Package pkg) {
    packageBuilderHelper.onLoadingComplete(pkg);
  }

  /**
   * Constructs a Package instance, evaluates the BUILD-file AST inside the
   * build environment, and populates the package with Rule instances as it
   * goes.  As with most programming languages, evaluation stops when an
   * exception is encountered: no further rules after the point of failure will
   * be constructed.  We assume that rules constructed before the point of
   * failure are valid; this assumption is not entirely correct, since a
   * "vardef" after a rule declaration can affect the behavior of that rule.
   *
   * <p>Rule attribute checking is performed during evaluation. Each attribute
   * must conform to the type specified for that <i>(rule class, attribute
   * name)</i> pair.  Errors reported at this stage include: missing value for
   * mandatory attribute, value of wrong type.  Such error cause Rule
   * construction to be aborted, so the resulting package will have missing
   * members.
   *
   * @see PackageFactory#PackageFactory
   */
  @VisibleForTesting // used by PackageFactoryApparatus
  public Package.Builder evaluateBuildFile(
      Package externalPkg,
      PackageIdentifier packageId,
      BuildFileAST buildFileAST,
      Path buildFilePath,
      Globber globber,
      Iterable<Event> pastEvents,
      RuleVisibility defaultVisibility,
      boolean containsError,
      MakeEnvironment.Builder pkgMakeEnv,
      Map<String, Extension> imports,
      ImmutableList<Label> skylarkFileDependencies)
      throws InterruptedException {
    Package.Builder pkgBuilder = new Package.Builder(packageBuilderHelper.createFreshPackage(
        packageId, ruleClassProvider.getRunfilesPrefix()));
    StoredEventHandler eventHandler = new StoredEventHandler();

    try (Mutability mutability = Mutability.create("package %s", packageId)) {
      Environment pkgEnv = Environment.builder(mutability)
          .setGlobals(Environment.BUILD)
          .setEventHandler(eventHandler)
          .setImportedExtensions(imports)
          .setToolsRepository(ruleClassProvider.getToolsRepository())
          .setPhase(Phase.LOADING)
          .build();

      pkgBuilder.setFilename(buildFilePath)
          .setMakeEnv(pkgMakeEnv)
          .setDefaultVisibility(defaultVisibility)
          // "defaultVisibility" comes from the command line. Let's give the BUILD file a chance to
          // set default_visibility once, be reseting the PackageBuilder.defaultVisibilitySet flag.
          .setDefaultVisibilitySet(false)
          .setSkylarkFileDependencies(skylarkFileDependencies)
          .setWorkspaceName(externalPkg.getWorkspaceName());

      Event.replayEventsOn(eventHandler, pastEvents);

      // Stuff that closes over the package context:
      PackageContext context =
          new PackageContext(
              pkgBuilder, globber, eventHandler, ruleFactory.getAttributeContainerFactory());
      buildPkgEnv(pkgEnv, context, ruleFactory);
      pkgEnv.setupDynamic(PKG_CONTEXT, context);
      pkgEnv.setupDynamic(Runtime.PKG_NAME, packageId.getPackageFragment().getPathString());
      pkgEnv.setupDynamic(Runtime.REPOSITORY_NAME, packageId.getRepository().toString());

      if (containsError) {
        pkgBuilder.setContainsErrors();
      }

      if (!validatePackageIdentifier(packageId, buildFileAST.getLocation(), eventHandler)) {
        pkgBuilder.setContainsErrors();
      }

      if (!validateAssignmentStatements(pkgEnv, buildFileAST, eventHandler)) {
        pkgBuilder.setContainsErrors();
      }

      if (buildFileAST.containsErrors()) {
        pkgBuilder.setContainsErrors();
      }

      // TODO(bazel-team): (2009) the invariant "if errors are reported, mark the package
      // as containing errors" is strewn all over this class.  Refactor to use an
      // event sensor--and see if we can simplify the calling code in
      // createPackage().
      if (!buildFileAST.exec(pkgEnv, eventHandler)) {
        pkgBuilder.setContainsErrors();
      }
    }

    pkgBuilder.addEvents(eventHandler.getEvents());
    return pkgBuilder;
  }

  /**
   * Visit all targets and expand the globs in parallel.
   */
  private void prefetchGlobs(PackageIdentifier packageId, BuildFileAST buildFileAST,
      boolean wasPreprocessed, Path buildFilePath, Globber globber,
      RuleVisibility defaultVisibility, MakeEnvironment.Builder pkgMakeEnv)
      throws InterruptedException {
    if (wasPreprocessed && preprocessorFactory.considersGlobs()) {
      // All the globs have either already been evaluated and they aren't in the ast anymore, or
      // they are in the ast but the globber has been evaluating them lazily and so there is no
      // point in prefetching them again.
      return;
    }
    // TODO(bazel-team): It may be wasteful to evaluate the BUILD file here, only to throw away the
    // result. It may be better to first scan the ast and see if there are even possibly any globs
    // at all. Additionally, it's wasteful to execute Skylark code that cannot invoke globs. So one
    // strategy would be to crawl the ast and tag statements whose execution cannot involve globs -
    // these can be executed and their impact on the resulting package can be saved.
    try (Mutability mutability = Mutability.create("prefetchGlobs for %s", packageId)) {
      Environment pkgEnv = Environment.builder(mutability)
          .setGlobals(Environment.BUILD)
          .setEventHandler(NullEventHandler.INSTANCE)
          .setToolsRepository(ruleClassProvider.getToolsRepository())
          .setPhase(Phase.LOADING)
          .build();

      Package.Builder pkgBuilder = new Package.Builder(packageBuilderHelper.createFreshPackage(
          packageId, ruleClassProvider.getRunfilesPrefix()));

      pkgBuilder.setFilename(buildFilePath)
          .setMakeEnv(pkgMakeEnv)
          .setDefaultVisibility(defaultVisibility)
          // "defaultVisibility" comes from the command line. Let's give the BUILD file a chance to
          // set default_visibility once, be reseting the PackageBuilder.defaultVisibilitySet flag.
          .setDefaultVisibilitySet(false);

      // Stuff that closes over the package context:
      PackageContext context =
          new PackageContext(
              pkgBuilder,
              globber,
              NullEventHandler.INSTANCE,
              ruleFactory.getAttributeContainerFactory());
      buildPkgEnv(pkgEnv, context, ruleFactory);
      try {
        pkgEnv.update("glob", newGlobFunction.apply(context, /*async=*/true));
        // The Fileset function is heavyweight in that it can run glob(). Avoid this during the
        // preloading phase.
        pkgEnv.update("FilesetEntry", Runtime.NONE);
      } catch (EvalException e) {
        throw new AssertionError(e);
      }
      buildFileAST.exec(pkgEnv, NullEventHandler.INSTANCE);
    }
  }


  /**
   * Tests a build AST to ensure that it contains no assignment statements that
   * redefine built-in build rules.
   *
   * @param pkgEnv a package environment initialized with all of the built-in
   *        build rules
   * @param ast the build file AST to be tested
   * @param eventHandler a eventHandler where any errors should be logged
   * @return true if the build file contains no redefinitions of built-in
   *         functions
   */
  // TODO(bazel-team): Remove this check. It should be moved to LValue.assign
  private static boolean validateAssignmentStatements(
      Environment pkgEnv, BuildFileAST ast, EventHandler eventHandler) {
    for (Statement stmt : ast.getStatements()) {
      if (stmt instanceof AssignmentStatement) {
        Expression lvalue = ((AssignmentStatement) stmt).getLValue().getExpression();
        if (!(lvalue instanceof Identifier)) {
          continue;
        }
        String target = ((Identifier) lvalue).getName();
        if (pkgEnv.lookup(target, null) != null) {
          eventHandler.handle(Event.error(stmt.getLocation(), "Reassignment of builtin build "
              + "function '" + target + "' not permitted"));
          return false;
        }
      }
    }
    return true;
  }

  // Reports an error and returns false iff package identifier was illegal.
  private static boolean validatePackageIdentifier(PackageIdentifier packageId, Location location,
      EventHandler eventHandler) {
    String error = LabelValidator.validatePackageName(packageId.getPackageFragment().toString());
    if (error != null) {
      eventHandler.handle(Event.error(location, error));
      return false; // Invalid package name 'foo'
    }
    return true;
  }

  static {
    SkylarkSignatureProcessor.configureSkylarkFunctions(PackageFactory.class);
  }

  /** Empty EnvironmentExtension */
  public static class EmptyEnvironmentExtension implements EnvironmentExtension {
    @Override
    public void update(Environment environment) {}

    @Override
    public void updateWorkspace(Environment environment) {}

    @Override
    public Iterable<PackageArgument<?>> getPackageArguments() {
      return ImmutableList.of();
    }

    @Override
    public ImmutableList<BaseFunction> nativeModuleFunctions() {
      return ImmutableList.<BaseFunction>of();
    }
  }
}