| // Copyright 2014 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| package com.google.devtools.build.lib.skyframe; |
| |
| import com.google.common.base.Preconditions; |
| import com.google.devtools.build.lib.actions.FileStateValue; |
| import com.google.devtools.build.lib.actions.FileValue; |
| import com.google.devtools.build.lib.analysis.BlazeDirectories; |
| import com.google.devtools.build.lib.cmdline.LabelConstants; |
| import com.google.devtools.build.lib.cmdline.RepositoryName; |
| import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; |
| import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; |
| import com.google.devtools.build.lib.pkgcache.PathPackageLocator; |
| import com.google.devtools.build.lib.rules.repository.ManagedDirectoriesKnowledge; |
| import com.google.devtools.build.lib.rules.repository.RepositoryFunction; |
| import com.google.devtools.build.lib.util.Pair; |
| import com.google.devtools.build.lib.vfs.Path; |
| import com.google.devtools.build.lib.vfs.RootedPath; |
| import com.google.devtools.build.skyframe.SkyFunction; |
| import java.io.IOException; |
| import java.util.concurrent.atomic.AtomicInteger; |
| import java.util.concurrent.atomic.AtomicReference; |
| import java.util.logging.Logger; |
| |
| /** Common utilities for dealing with paths outside the package roots. */ |
| public class ExternalFilesHelper { |
| private static final boolean IN_TEST = System.getenv("TEST_TMPDIR") != null; |
| |
| private static final Logger logger = Logger.getLogger(ExternalFilesHelper.class.getName()); |
| |
| private final AtomicReference<PathPackageLocator> pkgLocator; |
| private final ExternalFileAction externalFileAction; |
| private final BlazeDirectories directories; |
| private final int maxNumExternalFilesToLog; |
| private final AtomicInteger numExternalFilesLogged = new AtomicInteger(0); |
| |
| // These variables are set to true from multiple threads, but only read in the main thread. |
| // So volatility or an AtomicBoolean is not needed. |
| private boolean anyOutputFilesSeen = false; |
| private boolean anyNonOutputExternalFilesSeen = false; |
| |
| private final ManagedDirectoriesKnowledge managedDirectoriesKnowledge; |
| |
| private ExternalFilesHelper( |
| AtomicReference<PathPackageLocator> pkgLocator, |
| ExternalFileAction externalFileAction, |
| BlazeDirectories directories, |
| int maxNumExternalFilesToLog, |
| ManagedDirectoriesKnowledge managedDirectoriesKnowledge) { |
| this.pkgLocator = pkgLocator; |
| this.externalFileAction = externalFileAction; |
| this.directories = directories; |
| this.maxNumExternalFilesToLog = maxNumExternalFilesToLog; |
| this.managedDirectoriesKnowledge = managedDirectoriesKnowledge; |
| } |
| |
| public static ExternalFilesHelper create( |
| AtomicReference<PathPackageLocator> pkgLocator, |
| ExternalFileAction externalFileAction, |
| BlazeDirectories directories, |
| ManagedDirectoriesKnowledge managedDirectoriesKnowledge) { |
| return IN_TEST |
| ? createForTesting(pkgLocator, externalFileAction, directories, managedDirectoriesKnowledge) |
| : new ExternalFilesHelper( |
| pkgLocator, |
| externalFileAction, |
| directories, |
| /*maxNumExternalFilesToLog=*/ 100, |
| managedDirectoriesKnowledge); |
| } |
| |
| public static ExternalFilesHelper createForTesting( |
| AtomicReference<PathPackageLocator> pkgLocator, |
| ExternalFileAction externalFileAction, |
| BlazeDirectories directories) { |
| return createForTesting( |
| pkgLocator, |
| externalFileAction, |
| directories, |
| ManagedDirectoriesKnowledge.NO_MANAGED_DIRECTORIES); |
| } |
| |
| private static ExternalFilesHelper createForTesting( |
| AtomicReference<PathPackageLocator> pkgLocator, |
| ExternalFileAction externalFileAction, |
| BlazeDirectories directories, |
| ManagedDirectoriesKnowledge managedDirectoriesKnowledge) { |
| return new ExternalFilesHelper( |
| pkgLocator, |
| externalFileAction, |
| directories, |
| // These log lines are mostly spam during unit and integration tests. |
| /*maxNumExternalFilesToLog=*/ 0, |
| managedDirectoriesKnowledge); |
| } |
| |
| |
| /** |
| * The action to take when an external path is encountered. See {@link FileType} for the |
| * definition of "external". |
| */ |
| public enum ExternalFileAction { |
| /** |
| * For paths of type {@link FileType#EXTERNAL_REPO}, introduce a Skyframe dependency on the |
| * 'external' package. |
| * |
| * <p>This is the default for Bazel, since it's required for correctness of the external |
| * repositories feature. |
| */ |
| DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, |
| |
| /** |
| * For paths of type {@link FileType#EXTERNAL} or {@link FileType#OUTPUT}, assume the path does |
| * not exist and will never exist. |
| */ |
| ASSUME_NON_EXISTENT_AND_IMMUTABLE_FOR_EXTERNAL_PATHS, |
| } |
| |
| /** Classification of a path encountered by Bazel. */ |
| public enum FileType { |
| /** A path inside the package roots. */ |
| INTERNAL, |
| |
| /** |
| * A non {@link #EXTERNAL_REPO} or {@link #EXTERNAL_IN_MANAGED_DIRECTORY} path outside the |
| * package roots about which we may make no other assumptions. |
| */ |
| EXTERNAL, |
| |
| /** |
| * A path in Bazel's output tree that's a proper output of an action (*not* a source file in an |
| * external repository). Such files are theoretically mutable, but certain Bazel flags may tell |
| * Bazel to assume these paths are immutable. |
| * |
| * <p>Note that {@link ExternalFilesHelper#maybeHandleExternalFile} is only used for {@link |
| * FileStateValue} and {@link DirectoryListingStateValue}, and also note that output files do |
| * not normally have corresponding {@link FileValue} instances (and thus also {@link |
| * FileStateValue} instances) in the Skyframe graph ({@link ArtifactFunction} only uses {@link |
| * FileValue}s for source files). But {@link FileStateValue}s for output files can still make |
| * their way into the Skyframe graph if e.g. a source file is a symlink to an output file. |
| */ |
| // TODO(nharmata): Consider an alternative design where we have an OutputFileDiffAwareness. This |
| // could work but would first require that we clean up all RootedPath usage. |
| OUTPUT, |
| |
| /** |
| * A path in the part of Bazel's output tree that contains (/ symlinks to) to external |
| * repositories. Every such path under the external repository is generated by the execution of |
| * the corresponding repository rule, so these paths should not be cached by Skyframe before the |
| * RepositoryDirectoryValue is computed. |
| */ |
| EXTERNAL_REPO, |
| |
| /** |
| * A path is under one of the managed directories. Managed directories are user-owned |
| * directories, which can be incrementally updated by repository rules, so that the updated |
| * files are visible for the actions in the same build. |
| * |
| * <p>Every such path under the managed directory is generated or updated by the execution of |
| * the corresponding repository rule, so these paths should not be cached by Skyframe before the |
| * RepositoryDirectoryValue is computed. {@link ManagedDirectoriesKnowledge} |
| */ |
| EXTERNAL_IN_MANAGED_DIRECTORY, |
| } |
| |
| /** |
| * Thrown by {@link #maybeHandleExternalFile} when an applicable path is processed (see |
| * {@link ExternalFileAction#ASSUME_NON_EXISTENT_AND_IMMUTABLE_FOR_EXTERNAL_PATHS}. |
| */ |
| static class NonexistentImmutableExternalFileException extends Exception { |
| } |
| |
| static class ExternalFilesKnowledge { |
| final boolean anyOutputFilesSeen; |
| final boolean anyNonOutputExternalFilesSeen; |
| |
| private ExternalFilesKnowledge(boolean anyOutputFilesSeen, |
| boolean anyNonOutputExternalFilesSeen) { |
| this.anyOutputFilesSeen = anyOutputFilesSeen; |
| this.anyNonOutputExternalFilesSeen = anyNonOutputExternalFilesSeen; |
| } |
| } |
| |
| @ThreadCompatible |
| ExternalFilesKnowledge getExternalFilesKnowledge() { |
| return new ExternalFilesKnowledge(anyOutputFilesSeen, anyNonOutputExternalFilesSeen); |
| } |
| |
| @ThreadCompatible |
| void setExternalFilesKnowledge(ExternalFilesKnowledge externalFilesKnowledge) { |
| anyOutputFilesSeen = externalFilesKnowledge.anyOutputFilesSeen; |
| anyNonOutputExternalFilesSeen = externalFilesKnowledge.anyNonOutputExternalFilesSeen; |
| } |
| |
| ExternalFilesHelper cloneWithFreshExternalFilesKnowledge() { |
| return new ExternalFilesHelper( |
| pkgLocator, |
| externalFileAction, |
| directories, |
| maxNumExternalFilesToLog, |
| managedDirectoriesKnowledge); |
| } |
| |
| public FileType getAndNoteFileType(RootedPath rootedPath) { |
| return getFileTypeAndRepository(rootedPath).getFirst(); |
| } |
| |
| private Pair<FileType, RepositoryName> getFileTypeAndRepository(RootedPath rootedPath) { |
| RepositoryName repositoryName = |
| managedDirectoriesKnowledge.getOwnerRepository(rootedPath.getRootRelativePath()); |
| if (repositoryName != null) { |
| anyNonOutputExternalFilesSeen = true; |
| return Pair.of(FileType.EXTERNAL_IN_MANAGED_DIRECTORY, repositoryName); |
| } |
| FileType fileType = detectFileType(rootedPath); |
| if (FileType.EXTERNAL == fileType || FileType.EXTERNAL_REPO == fileType) { |
| anyNonOutputExternalFilesSeen = true; |
| } |
| if (FileType.OUTPUT == fileType) { |
| anyOutputFilesSeen = true; |
| } |
| return Pair.of(fileType, null); |
| } |
| |
| private FileType detectFileType(RootedPath rootedPath) { |
| PathPackageLocator packageLocator = pkgLocator.get(); |
| if (packageLocator.getPathEntries().contains(rootedPath.getRoot())) { |
| return FileType.INTERNAL; |
| } |
| // The outputBase may be null if we're not actually running a build. |
| Path outputBase = packageLocator.getOutputBase(); |
| if (outputBase == null) { |
| anyNonOutputExternalFilesSeen = true; |
| return FileType.EXTERNAL; |
| } |
| if (rootedPath.asPath().startsWith(outputBase)) { |
| Path externalRepoDir = outputBase.getRelative(LabelConstants.EXTERNAL_PACKAGE_NAME); |
| if (rootedPath.asPath().startsWith(externalRepoDir)) { |
| anyNonOutputExternalFilesSeen = true; |
| return FileType.EXTERNAL_REPO; |
| } else { |
| anyOutputFilesSeen = true; |
| return FileType.OUTPUT; |
| } |
| } |
| anyNonOutputExternalFilesSeen = true; |
| return FileType.EXTERNAL; |
| } |
| |
| /** |
| * If this instance is configured with {@link |
| * ExternalFileAction#DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS} and {@code rootedPath} isn't |
| * under a package root then this adds a dependency on the //external package. If the action is |
| * {@link ExternalFileAction#ASSUME_NON_EXISTENT_AND_IMMUTABLE_FOR_EXTERNAL_PATHS}, it will throw |
| * a {@link NonexistentImmutableExternalFileException} instead. |
| */ |
| @ThreadSafe |
| FileType maybeHandleExternalFile( |
| RootedPath rootedPath, boolean isDirectory, SkyFunction.Environment env) |
| throws NonexistentImmutableExternalFileException, IOException, InterruptedException { |
| Pair<FileType, RepositoryName> pair = getFileTypeAndRepository(rootedPath); |
| |
| FileType fileType = Preconditions.checkNotNull(pair.getFirst()); |
| switch (fileType) { |
| case EXTERNAL_IN_MANAGED_DIRECTORY: |
| Preconditions.checkState( |
| externalFileAction == ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, |
| externalFileAction); |
| RepositoryFunction.addManagedDirectoryDependencies(pair.getSecond(), env); |
| break; |
| case INTERNAL: |
| break; |
| case EXTERNAL: |
| if (numExternalFilesLogged.incrementAndGet() < maxNumExternalFilesToLog) { |
| logger.info("Encountered an external path " + rootedPath); |
| } |
| // fall through |
| case OUTPUT: |
| if (externalFileAction |
| == ExternalFileAction.ASSUME_NON_EXISTENT_AND_IMMUTABLE_FOR_EXTERNAL_PATHS) { |
| throw new NonexistentImmutableExternalFileException(); |
| } |
| break; |
| case EXTERNAL_REPO: |
| Preconditions.checkState( |
| externalFileAction == ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, |
| externalFileAction); |
| RepositoryFunction.addExternalFilesDependencies(rootedPath, isDirectory, directories, env); |
| break; |
| } |
| return fileType; |
| } |
| } |