blob: 1fcbc4e0e71163eb1ad74cd494c2396055d479f4 [file] [log] [blame]
// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
import com.google.common.base.Preconditions;
import com.google.devtools.build.lib.actions.FileStateValue;
import com.google.devtools.build.lib.actions.FileValue;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.cmdline.LabelConstants;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
import com.google.devtools.build.lib.rules.repository.ManagedDirectoriesKnowledge;
import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
import com.google.devtools.build.lib.util.Pair;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.skyframe.SkyFunction;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Logger;
/** Common utilities for dealing with paths outside the package roots. */
public class ExternalFilesHelper {
private static final boolean IN_TEST = System.getenv("TEST_TMPDIR") != null;
private static final Logger logger = Logger.getLogger(ExternalFilesHelper.class.getName());
private final AtomicReference<PathPackageLocator> pkgLocator;
private final ExternalFileAction externalFileAction;
private final BlazeDirectories directories;
private final int maxNumExternalFilesToLog;
private final AtomicInteger numExternalFilesLogged = new AtomicInteger(0);
// These variables are set to true from multiple threads, but only read in the main thread.
// So volatility or an AtomicBoolean is not needed.
private boolean anyOutputFilesSeen = false;
private boolean anyNonOutputExternalFilesSeen = false;
private final ManagedDirectoriesKnowledge managedDirectoriesKnowledge;
private ExternalFilesHelper(
AtomicReference<PathPackageLocator> pkgLocator,
ExternalFileAction externalFileAction,
BlazeDirectories directories,
int maxNumExternalFilesToLog,
ManagedDirectoriesKnowledge managedDirectoriesKnowledge) {
this.pkgLocator = pkgLocator;
this.externalFileAction = externalFileAction;
this.directories = directories;
this.maxNumExternalFilesToLog = maxNumExternalFilesToLog;
this.managedDirectoriesKnowledge = managedDirectoriesKnowledge;
}
public static ExternalFilesHelper create(
AtomicReference<PathPackageLocator> pkgLocator,
ExternalFileAction externalFileAction,
BlazeDirectories directories,
ManagedDirectoriesKnowledge managedDirectoriesKnowledge) {
return IN_TEST
? createForTesting(pkgLocator, externalFileAction, directories, managedDirectoriesKnowledge)
: new ExternalFilesHelper(
pkgLocator,
externalFileAction,
directories,
/*maxNumExternalFilesToLog=*/ 100,
managedDirectoriesKnowledge);
}
public static ExternalFilesHelper createForTesting(
AtomicReference<PathPackageLocator> pkgLocator,
ExternalFileAction externalFileAction,
BlazeDirectories directories) {
return createForTesting(
pkgLocator,
externalFileAction,
directories,
ManagedDirectoriesKnowledge.NO_MANAGED_DIRECTORIES);
}
private static ExternalFilesHelper createForTesting(
AtomicReference<PathPackageLocator> pkgLocator,
ExternalFileAction externalFileAction,
BlazeDirectories directories,
ManagedDirectoriesKnowledge managedDirectoriesKnowledge) {
return new ExternalFilesHelper(
pkgLocator,
externalFileAction,
directories,
// These log lines are mostly spam during unit and integration tests.
/*maxNumExternalFilesToLog=*/ 0,
managedDirectoriesKnowledge);
}
/**
* The action to take when an external path is encountered. See {@link FileType} for the
* definition of "external".
*/
public enum ExternalFileAction {
/**
* For paths of type {@link FileType#EXTERNAL_REPO}, introduce a Skyframe dependency on the
* 'external' package.
*
* <p>This is the default for Bazel, since it's required for correctness of the external
* repositories feature.
*/
DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS,
/**
* For paths of type {@link FileType#EXTERNAL} or {@link FileType#OUTPUT}, assume the path does
* not exist and will never exist.
*/
ASSUME_NON_EXISTENT_AND_IMMUTABLE_FOR_EXTERNAL_PATHS,
}
/** Classification of a path encountered by Bazel. */
public enum FileType {
/** A path inside the package roots. */
INTERNAL,
/**
* A non {@link #EXTERNAL_REPO} or {@link #EXTERNAL_IN_MANAGED_DIRECTORY} path outside the
* package roots about which we may make no other assumptions.
*/
EXTERNAL,
/**
* A path in Bazel's output tree that's a proper output of an action (*not* a source file in an
* external repository). Such files are theoretically mutable, but certain Bazel flags may tell
* Bazel to assume these paths are immutable.
*
* <p>Note that {@link ExternalFilesHelper#maybeHandleExternalFile} is only used for {@link
* FileStateValue} and {@link DirectoryListingStateValue}, and also note that output files do
* not normally have corresponding {@link FileValue} instances (and thus also {@link
* FileStateValue} instances) in the Skyframe graph ({@link ArtifactFunction} only uses {@link
* FileValue}s for source files). But {@link FileStateValue}s for output files can still make
* their way into the Skyframe graph if e.g. a source file is a symlink to an output file.
*/
// TODO(nharmata): Consider an alternative design where we have an OutputFileDiffAwareness. This
// could work but would first require that we clean up all RootedPath usage.
OUTPUT,
/**
* A path in the part of Bazel's output tree that contains (/ symlinks to) to external
* repositories. Every such path under the external repository is generated by the execution of
* the corresponding repository rule, so these paths should not be cached by Skyframe before the
* RepositoryDirectoryValue is computed.
*/
EXTERNAL_REPO,
/**
* A path is under one of the managed directories. Managed directories are user-owned
* directories, which can be incrementally updated by repository rules, so that the updated
* files are visible for the actions in the same build.
*
* <p>Every such path under the managed directory is generated or updated by the execution of
* the corresponding repository rule, so these paths should not be cached by Skyframe before the
* RepositoryDirectoryValue is computed. {@link ManagedDirectoriesKnowledge}
*/
EXTERNAL_IN_MANAGED_DIRECTORY,
}
/**
* Thrown by {@link #maybeHandleExternalFile} when an applicable path is processed (see
* {@link ExternalFileAction#ASSUME_NON_EXISTENT_AND_IMMUTABLE_FOR_EXTERNAL_PATHS}.
*/
static class NonexistentImmutableExternalFileException extends Exception {
}
static class ExternalFilesKnowledge {
final boolean anyOutputFilesSeen;
final boolean anyNonOutputExternalFilesSeen;
private ExternalFilesKnowledge(boolean anyOutputFilesSeen,
boolean anyNonOutputExternalFilesSeen) {
this.anyOutputFilesSeen = anyOutputFilesSeen;
this.anyNonOutputExternalFilesSeen = anyNonOutputExternalFilesSeen;
}
}
@ThreadCompatible
ExternalFilesKnowledge getExternalFilesKnowledge() {
return new ExternalFilesKnowledge(anyOutputFilesSeen, anyNonOutputExternalFilesSeen);
}
@ThreadCompatible
void setExternalFilesKnowledge(ExternalFilesKnowledge externalFilesKnowledge) {
anyOutputFilesSeen = externalFilesKnowledge.anyOutputFilesSeen;
anyNonOutputExternalFilesSeen = externalFilesKnowledge.anyNonOutputExternalFilesSeen;
}
ExternalFilesHelper cloneWithFreshExternalFilesKnowledge() {
return new ExternalFilesHelper(
pkgLocator,
externalFileAction,
directories,
maxNumExternalFilesToLog,
managedDirectoriesKnowledge);
}
public FileType getAndNoteFileType(RootedPath rootedPath) {
return getFileTypeAndRepository(rootedPath).getFirst();
}
private Pair<FileType, RepositoryName> getFileTypeAndRepository(RootedPath rootedPath) {
RepositoryName repositoryName =
managedDirectoriesKnowledge.getOwnerRepository(rootedPath.getRootRelativePath());
if (repositoryName != null) {
anyNonOutputExternalFilesSeen = true;
return Pair.of(FileType.EXTERNAL_IN_MANAGED_DIRECTORY, repositoryName);
}
FileType fileType = detectFileType(rootedPath);
if (FileType.EXTERNAL == fileType || FileType.EXTERNAL_REPO == fileType) {
anyNonOutputExternalFilesSeen = true;
}
if (FileType.OUTPUT == fileType) {
anyOutputFilesSeen = true;
}
return Pair.of(fileType, null);
}
private FileType detectFileType(RootedPath rootedPath) {
PathPackageLocator packageLocator = pkgLocator.get();
if (packageLocator.getPathEntries().contains(rootedPath.getRoot())) {
return FileType.INTERNAL;
}
// The outputBase may be null if we're not actually running a build.
Path outputBase = packageLocator.getOutputBase();
if (outputBase == null) {
anyNonOutputExternalFilesSeen = true;
return FileType.EXTERNAL;
}
if (rootedPath.asPath().startsWith(outputBase)) {
Path externalRepoDir = outputBase.getRelative(LabelConstants.EXTERNAL_PACKAGE_NAME);
if (rootedPath.asPath().startsWith(externalRepoDir)) {
anyNonOutputExternalFilesSeen = true;
return FileType.EXTERNAL_REPO;
} else {
anyOutputFilesSeen = true;
return FileType.OUTPUT;
}
}
anyNonOutputExternalFilesSeen = true;
return FileType.EXTERNAL;
}
/**
* If this instance is configured with {@link
* ExternalFileAction#DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS} and {@code rootedPath} isn't
* under a package root then this adds a dependency on the //external package. If the action is
* {@link ExternalFileAction#ASSUME_NON_EXISTENT_AND_IMMUTABLE_FOR_EXTERNAL_PATHS}, it will throw
* a {@link NonexistentImmutableExternalFileException} instead.
*/
@ThreadSafe
FileType maybeHandleExternalFile(
RootedPath rootedPath, boolean isDirectory, SkyFunction.Environment env)
throws NonexistentImmutableExternalFileException, IOException, InterruptedException {
Pair<FileType, RepositoryName> pair = getFileTypeAndRepository(rootedPath);
FileType fileType = Preconditions.checkNotNull(pair.getFirst());
switch (fileType) {
case EXTERNAL_IN_MANAGED_DIRECTORY:
Preconditions.checkState(
externalFileAction == ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS,
externalFileAction);
RepositoryFunction.addManagedDirectoryDependencies(pair.getSecond(), env);
break;
case INTERNAL:
break;
case EXTERNAL:
if (numExternalFilesLogged.incrementAndGet() < maxNumExternalFilesToLog) {
logger.info("Encountered an external path " + rootedPath);
}
// fall through
case OUTPUT:
if (externalFileAction
== ExternalFileAction.ASSUME_NON_EXISTENT_AND_IMMUTABLE_FOR_EXTERNAL_PATHS) {
throw new NonexistentImmutableExternalFileException();
}
break;
case EXTERNAL_REPO:
Preconditions.checkState(
externalFileAction == ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS,
externalFileAction);
RepositoryFunction.addExternalFilesDependencies(rootedPath, isDirectory, directories, env);
break;
}
return fileType;
}
}