blob: b37676d2421566ac7bdfabbf0bb724bbb22ce482 [file] [log] [blame]
// Copyright 2018 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.includescanning;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionExecutionMetadata;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ArtifactFactory;
import com.google.devtools.build.lib.actions.ArtifactRoot;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.MissingDepException;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor;
import com.google.devtools.build.lib.concurrent.ErrorClassifier;
import com.google.devtools.build.lib.concurrent.ThreadSafety;
import com.google.devtools.build.lib.includescanning.IncludeParser.Hints;
import com.google.devtools.build.lib.includescanning.IncludeParser.Inclusion;
import com.google.devtools.build.lib.includescanning.IncludeParser.Inclusion.Kind;
import com.google.devtools.build.lib.rules.cpp.IncludeScanner;
import com.google.devtools.build.lib.vfs.IORuntimeException;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.skyframe.SkyFunction;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
/**
* C include scanner. Quickly scans C/C++ source files to determine the bounding set of transitively
* referenced include files.
*
* <p>Maintains caches for parses and search-matches for performance.
*
* <pre>
* TODO(bazel-team): (2009) Currently does not evaluate preprocessor symbols, so computed includes
* are ignored.
* TODO(bazel-team): (2009) Does not handle multiline block comments preceding or around an #include
* </pre>
*/
public class LegacyIncludeScanner implements IncludeScanner {
private static final class ArtifactWithInclusionContext {
private final Artifact artifact;
private final Kind contextKind;
private final int contextPathPos;
private ArtifactWithInclusionContext(Artifact artifact, Kind contextKind, int contextPathPos) {
this.artifact = artifact;
this.contextKind = contextKind;
this.contextPathPos = contextPathPos;
}
@Override
public int hashCode() {
return contextPathPos + 37 * Objects.hash(contextKind, artifact);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (!(obj instanceof ArtifactWithInclusionContext)) {
return false;
}
ArtifactWithInclusionContext that = (ArtifactWithInclusionContext) obj;
return this.contextKind == that.contextKind
&& this.contextPathPos == that.contextPathPos
&& this.artifact.equals(that.artifact);
}
}
/**
* A cache of inclusion lookups, taking care to avoid spurious caching related
* to generated headers / source files.
*/
@ThreadSafety.ThreadSafe
private class InclusionCache {
private final ConcurrentMap<InclusionWithContext, LocateOnPathResult> cache =
new ConcurrentHashMap<>();
/**
* Locates an included file along the search paths. The result is cacheable.
*
* @param inclusion the inclusion to locate
* @param pathToLegalOutputArtifact generated files which may be reached during scanning
* @param onlyCheckGenerated if true, only search for generated output files
* @return a tuple of the found file, the position of the respective include path entry on the
* search path (or null if no matching file was found), and whether the scan touched illegal
* output files
*/
private LocateOnPathResult locateOnPaths(
InclusionWithContext inclusion,
Map<PathFragment, Artifact> pathToLegalOutputArtifact,
boolean onlyCheckGenerated) {
PathFragment name = inclusion.getInclusion().pathFragment;
// For #include_next directives we start searching on the include path where
// we found the previous inclusion.
int searchStart = inclusion.getInclusion().kind.isNext() ? inclusion.getContextPathPos() : 0;
// Search the header on the remaining paths.
List<PathFragment> paths =
inclusion.getContextKind() == Kind.QUOTE ? quoteIncludePaths : includePaths;
boolean viewedIllegalOutput = false;
for (int i = searchStart; i < paths.size(); ++i) {
PathFragment fileFragment = paths.get(i).getRelative(name);
if (fileFragment.containsUplevelReferences()) {
// TODO(janakr): This branch shouldn't be necessary: we should be able to filter such
// inclusions out unconditionally.
// Deal with fragments that escape the execroot. They most likely come right back in.
Path execRootRelativePath = execRoot.getRelative(fileFragment);
if (execRootRelativePath.startsWith(execRoot)) {
// Common case: transform #include "../execroot/foo.h" into #include "foo.h"
fileFragment = execRootRelativePath.relativeTo(execRoot);
} else {
// Ugh: successfully escaped the exec root. It's their funeral.
fileFragment = execRootRelativePath.asFragment();
}
// This can happen when we are processing Windows paths with backslashes on Unix,
// since we do not do any #ifdef processing.
// We can safely discard these here.
if (fileFragment.containsUplevelReferences()) {
continue;
}
}
if (onlyCheckGenerated && !isRealOutputFile(fileFragment)) {
continue;
}
viewedIllegalOutput =
viewedIllegalOutput
|| isIllegalOutputFile(fileFragment, pathToLegalOutputArtifact.keySet());
boolean isOutputDirectory = fileFragment.startsWith(outputPathFragment);
if (!isFile(fileFragment, name, !isOutputDirectory, pathToLegalOutputArtifact.keySet())) {
continue;
}
Artifact artifact;
if (isOutputDirectory) {
// May be a normal output file or an inc_library header.
artifact = pathToLegalOutputArtifact.get(fileFragment);
if (artifact == null) {
// This happens if an included file exists in a cc_inc_library's output directory,
// but is not an output of the cc_inc_library. This can happen if, for instance, the
// definition of the cc_inc_library is changed to output different files, but the
// source file's includes don't change.
// Often, such an include is conditional, and so failing to find it here will not
// lead to problems. If this include is actually needed for compilation, then we will
// emit a somewhat unhelpful error message of a missing file, rather than the more
// helpful one of an illegal include, but it's hard to emit the illegal include
// message consistently, and this is a rare occurrence in any case.
return LocateOnPathResult.createNotFound(viewedIllegalOutput);
}
} else if (!fileFragment.isAbsolute()) {
artifact = artifactFactory.resolveSourceArtifact(fileFragment, RepositoryName.MAIN);
if (artifact == null) {
// There was a real file, but we couldn't resolve it, probably because it belonged to
// a package that wasn't actually loaded this build, so user cannot refer to files in
// that package.
continue;
}
} else {
// This file is given with an absolute path. We will error out after transitive scanning
// of the top-level source is finished unless this corresponds to a built-in include
// directory, and will ignore this artifact in any case, but track it here so that its
// includes can be processed.
artifact = artifactFactory.getSourceArtifact(fileFragment, absoluteRoot);
}
// +1 to account for the virtual entry for relative includes.
return LocateOnPathResult.create(artifact, i + 1, viewedIllegalOutput);
}
// Not found.
return LocateOnPathResult.createNotFound(viewedIllegalOutput);
}
/**
* Locates an included file along the search paths.
*
* @param inclusion the inclusion to locate
* @param pathToLegalOutputArtifact generated files which may be reached during scanning
* @return a LocateOnPathResult
*/
public LocateOnPathResult lookup(
InclusionWithContext inclusion, Map<PathFragment, Artifact> pathToLegalOutputArtifact) {
LocateOnPathResult result =
cache.computeIfAbsent(
inclusion, key -> locateOnPaths(key, pathToLegalOutputArtifact, false));
// It is not safe to cache lookups which viewed illegal output files. Their nonexistence do
// not imply nonexistence for actions using the same include scanner, but executed later on.
// See bug 2097998. For performance reasons, take a small shortcut: only avoid caching when
// the path lookup result from locateOnPaths() is empty.
if (result.path != null || !result.viewedIllegalOutputFile) {
return result;
}
result = locateOnPaths(inclusion, pathToLegalOutputArtifact, true);
if (result.path != null || !result.viewedIllegalOutputFile) {
// In this case, the result is now cachable either because a file has been found or
// because there are no more illegal output files. This is rare in practice. Avoid
// creating a future and modifying the cache in the common case.
cache.put(inclusion, result);
}
return result;
}
}
private static class LocateOnPathResult {
private static final LocateOnPathResult NOT_FOUND_VIEWED_ILLEGAL =
new LocateOnPathResult(null, -1, true);
private static final LocateOnPathResult NOT_FOUND_NO_VIEWED_ILLEGAL =
new LocateOnPathResult(null, -1, false);
private final Artifact path;
private final int includePosition;
private final boolean viewedIllegalOutputFile;
private LocateOnPathResult(
Artifact path, int includePosition, boolean viewedIllegalOutputFile) {
this.path = path;
this.includePosition = includePosition;
this.viewedIllegalOutputFile = viewedIllegalOutputFile;
}
static LocateOnPathResult create(
Artifact path, int includePosition, boolean viewedIllegalOutputFile) {
return new LocateOnPathResult(
Preconditions.checkNotNull(path), includePosition, viewedIllegalOutputFile);
}
static LocateOnPathResult createNotFound(boolean viewedIllegalOutputFile) {
return viewedIllegalOutputFile ? NOT_FOUND_VIEWED_ILLEGAL : NOT_FOUND_NO_VIEWED_ILLEGAL;
}
}
private final Path execRoot;
private final ArtifactFactory artifactFactory;
private final Supplier<SpawnIncludeScanner> spawnIncludeScannerSupplier;
/**
* Externally-scoped cache of file path => parsed inclusion set mappings. Saves us from having to
* parse files more than once, and can be shared by scanners with different search paths.
*/
private final ConcurrentMap<Artifact, Collection<Inclusion>> fileParseCache;
private final IncludeParser parser;
/**
* Search path for searching for all quoted "xyz.h" includes, composed of all the -iquote, -I and
* -isystem paths (in this order).
*/
private final ImmutableList<PathFragment> quoteIncludePaths;
/**
* Search path for searching for all includes, composed of all the -I and -isystem paths (in this
* order).
*/
private final List<PathFragment> includePaths;
private final PathFragment includeRootFragment;
private final PathFragment outputPathFragment;
private final Root absoluteRoot;
/**
* Scanner-scoped cache of inclusions with their resolved files and include path entries. This
* cache is specific to a given pair of search paths, and is thus scanner-local.
*
* <p>Each inclusion (name+type+context) is associated with its resolved file here, thus saving
* redundant path searches. The second entry of the pair is the include path entry on which the
* file was found.
*/
private final InclusionCache inclusionCache;
private final PathExistenceCache pathCache;
private final ExecutorService includePool;
// We are using this Random just for shuffling, so keep the order deterministic by hardcoding
// the seed.
private static final Random CONSTANT_SEED_RANDOM = new Random(88);
/**
* Constructs a new IncludeScanner
*
* @param cache externally scoped cache of file-path to inclusion-set mappings
* @param pathCache include path existence cache
* @param quoteIncludePaths the list of quote search path dirs (-iquote)
* @param includePaths the list of all other search path dirs (-I and -isystem)
*/
LegacyIncludeScanner(
IncludeParser parser,
ExecutorService includePool,
ConcurrentMap<Artifact, Collection<Inclusion>> cache,
PathExistenceCache pathCache,
List<PathFragment> quoteIncludePaths,
List<PathFragment> includePaths,
Path outputPath,
Path execRoot,
ArtifactFactory artifactFactory,
Supplier<SpawnIncludeScanner> spawnIncludeScannerSupplier) {
this.parser = parser;
this.includePool = includePool;
this.fileParseCache = cache;
this.pathCache = pathCache;
this.artifactFactory = Preconditions.checkNotNull(artifactFactory);
this.spawnIncludeScannerSupplier = spawnIncludeScannerSupplier;
this.quoteIncludePaths = ImmutableList.<PathFragment>builder()
.addAll(quoteIncludePaths)
.addAll(includePaths)
.build();
this.includePaths = ImmutableList.copyOf(includePaths);
this.inclusionCache = new InclusionCache();
this.execRoot = execRoot;
this.outputPathFragment = outputPath.relativeTo(execRoot);
this.includeRootFragment =
outputPathFragment.getRelative(BlazeDirectories.RELATIVE_INCLUDE_DIR);
this.absoluteRoot = Root.absoluteRoot(execRoot.getFileSystem());
}
/**
* Locates an included file relative to the including file. The result is not cacheable.
*
* @param inclusion the inclusion to locate
* @param includer the including file
* @return the resolved Path, or null if no file could be found
*/
private Artifact locateRelative(
Inclusion inclusion, Map<PathFragment, Artifact> legalOutputFiles, Artifact includer) {
if (inclusion.kind != Kind.QUOTE) {
return null;
}
PathFragment name = inclusion.pathFragment;
PathFragment execPath = includer.getExecPath().getParentDirectory().getRelative(name);
if (!isFile(execPath, name, includer.isSourceArtifact(), legalOutputFiles.keySet())) {
return null;
}
PathFragment parentDirectory = includer.getRootRelativePath().getParentDirectory();
PathFragment rootRelativePath = parentDirectory.getRelative(name);
if (rootRelativePath.containsUplevelReferences()) {
// An include cannot break out of a (package path) root via a relative inclusion. It should
// also not break out of the root and then come back into it -- who knows what hardcoded
// directory names there are in it.
return null;
}
if (legalOutputFiles.containsKey(execPath)) {
return legalOutputFiles.get(execPath);
}
ArtifactRoot root = includer.getRoot();
Artifact sourceArtifact =
artifactFactory.resolveSourceArtifactWithAncestor(
name, parentDirectory, root, RepositoryName.MAIN);
if (sourceArtifact == null) {
// If the name had up-level references, this path may not be under any package. Otherwise,
// we must have gotten an artifact, since it should be under the same package as the
// including artifact.
Preconditions.checkState(
name.containsUplevelReferences(),
"%s %s %s %s",
name,
parentDirectory,
rootRelativePath,
root);
}
return sourceArtifact;
}
/** Returns whether the given path exists in the filesystem. */
private boolean isFile(
PathFragment execPath, PathFragment includeAsWritten, boolean isSource,
Collection<PathFragment> legalOutputFiles) {
if (isRealOutputFile(execPath)) {
return legalOutputFiles.contains(execPath);
}
// TODO(djasper): This code path cannot be hit with isSource being false. Verify and add
// Preconditions check.
if (isSource && !execPath.isAbsolute() && execPath.endsWith(includeAsWritten)) {
// Verify that the directory of execPath exists as an optimization. Most includes are relative
// to the workspace and we'd like to avoid stat'ing every such include relative to every
// include path. If testing whether "a/b/c.h" is a file beneath the include path "e/f/",
// verify that "e/f/a" and "e/f/a/b" are valid directories (and cache the result).
int execPathSegments = execPath.segmentCount();
int nameSegments = includeAsWritten.segmentCount();
for (int i = execPathSegments - nameSegments + 1; i < execPathSegments; i++) {
if (!pathCache.directoryExists(execPath.subFragment(0, i))) {
return false;
}
}
}
return pathCache.fileExists(execPath, isSource);
}
@Override
public void process(
Artifact mainSource,
Collection<Artifact> sources,
IncludeScanningHeaderData includeScanningHeaderData,
List<String> cmdlineIncludes,
Set<Artifact> includes,
ActionExecutionMetadata actionExecutionMetadata,
ActionExecutionContext actionExecutionContext,
Artifact grepIncludes)
throws IOException, ExecException, InterruptedException {
try {
// Because our IncludeVisitor can dynamically switch to in-thread execution, we could get
// unlucky and run one of the async calls on the main thread. Catch and rethrow the runtime
// exceptions as the appropriate corresponding checked exception.
processInternal(
mainSource,
sources,
includeScanningHeaderData,
cmdlineIncludes,
includes,
actionExecutionMetadata,
actionExecutionContext,
grepIncludes);
} catch (IORuntimeException e) {
throw e.getCauseIOException();
} catch (ExecRuntimeException e) {
throw e.getRealCause();
} catch (InterruptedRuntimeException e) {
throw e.getRealCause();
}
}
private void processInternal(
Artifact mainSource,
Collection<Artifact> sources,
IncludeScanningHeaderData includeScanningHeaderData,
List<String> cmdlineIncludes,
Set<Artifact> includes,
ActionExecutionMetadata actionExecutionMetadata,
ActionExecutionContext actionExecutionContext,
Artifact grepIncludes)
throws IOException, ExecException, InterruptedException {
Preconditions.checkArgument(mainSource == null || sources.contains(mainSource),
"The main source '%s' is not part of '%s'", mainSource, sources);
ImmutableSet.Builder<Artifact> pathHints = null;
SkyFunction.Environment env = actionExecutionContext.getEnvironmentForDiscoveringInputs();
if (parser.getHints() != null) {
pathHints = ImmutableSet.builderWithExpectedSize(quoteIncludePaths.size());
Collection<Artifact> artifacts =
parser.getHints().getPathLevelHintedInclusions(quoteIncludePaths, env);
if (!env.valuesMissing()) {
pathHints.addAll(Preconditions.checkNotNull(artifacts, quoteIncludePaths));
}
}
if (env.valuesMissing()) {
throw new MissingDepException();
}
Set<ArtifactWithInclusionContext> visitedInclusions = Sets.newConcurrentHashSet();
IncludeVisitor visitor = new IncludeVisitor(includeScanningHeaderData.getModularHeaders());
try {
// Process cmd line includes, if specified.
if (mainSource != null && !cmdlineIncludes.isEmpty()) {
visitor.processCmdlineIncludes(
mainSource,
cmdlineIncludes,
grepIncludes,
includes,
includeScanningHeaderData.getPathToLegalOutputArtifact(),
actionExecutionMetadata,
actionExecutionContext,
visitedInclusions);
visitor.sync();
}
visitor.processBulkAsync(
sources,
includes,
includeScanningHeaderData.getPathToLegalOutputArtifact(),
actionExecutionMetadata,
actionExecutionContext,
visitedInclusions,
grepIncludes);
visitor.sync();
// Process include hints
// TODO(ulfjack): Make this code go away. Use the new hinted inclusions instead.
Hints hints = parser.getHints();
if (hints != null) {
// Follow "path" hints.
visitor.processBulkAsync(
pathHints.build(),
includes,
includeScanningHeaderData.getPathToLegalOutputArtifact(),
actionExecutionMetadata,
actionExecutionContext,
visitedInclusions,
grepIncludes);
// Follow "file" hints for the primary sources.
for (Artifact source : sources) {
visitor.processFileLevelHintsAsync(
hints,
source,
includes,
includeScanningHeaderData.getPathToLegalOutputArtifact(),
actionExecutionMetadata,
actionExecutionContext,
visitedInclusions,
grepIncludes);
}
visitor.sync();
// Follow "file" hints for all included headers, transitively.
Set<Artifact> frontier = includes;
while (!frontier.isEmpty()) {
Set<Artifact> adjacent = Sets.newConcurrentHashSet();
for (Artifact include : frontier) {
visitor.processFileLevelHintsAsync(
hints,
include,
adjacent,
includeScanningHeaderData.getPathToLegalOutputArtifact(),
actionExecutionMetadata,
actionExecutionContext,
visitedInclusions,
grepIncludes);
}
visitor.sync();
// Keep novel nodes as the next frontier.
for (Iterator<Artifact> iter = adjacent.iterator(); iter.hasNext(); ) {
if (!includes.add(iter.next())) {
iter.remove();
}
}
frontier = adjacent;
}
}
} catch (IOException | InterruptedException | ExecException | MissingDepException e) {
// Careful: Do not leak visitation threads if we have an exception in the initial thread.
visitor.sync();
throw e;
}
}
private static void checkForInterrupt(String operation, Object source)
throws InterruptedException {
// We require passing in the operation and the source Path / Artifact to avoid intermediate
// String operations. The include scanner is performance critical and this showed up in a
// profiler.
if (Thread.currentThread().isInterrupted()) {
throw new InterruptedException(
"Include scanning interrupted while " + operation + " " + source);
}
}
private boolean isIllegalOutputFile(
PathFragment includeFile, Collection<PathFragment> legalOutputFiles) {
return isRealOutputFile(includeFile) && !legalOutputFiles.contains(includeFile);
}
private boolean isRealOutputFile(PathFragment path) {
return path.startsWith(outputPathFragment) && !isIncPath(path);
}
private boolean isIncPath(PathFragment path) {
// See CreateIncSymlinkAction and where it's used: The symlink trees
// are always rooted at locations that fit the logic here.
return path.startsWith(includeRootFragment) && !path.equals(includeRootFragment);
}
/**
* Implements a potentially parallel traversal over source files using a
* thread pool shared across different IncludeScanner instances.
*/
private class IncludeVisitor extends AbstractQueueVisitor {
/** The set of headers known to be part of a C++ module. Scanning can stop here. */
private Set<Artifact> modularHeaders;
public IncludeVisitor(Set<Artifact> modularHeaders) {
super(
includePool,
/*shutdownOnCompletion=*/ false,
/*failFastOnException=*/ true,
ErrorClassifier.DEFAULT);
this.modularHeaders = modularHeaders;
}
/**
* Block for the completion of all outstanding visitations.
*/
public void sync() throws IOException, ExecException, InterruptedException {
try {
super.awaitQuiescence(true);
} catch (InterruptedException e) {
throw new InterruptedException("Interrupted during include visitation");
} catch (IORuntimeException e) {
throw e.getCauseIOException();
} catch (ExecRuntimeException e) {
throw e.getRealCause();
} catch (InterruptedRuntimeException e) {
throw e.getRealCause();
}
}
/**
* Processes a given file for includes and populates the provided set with the visited includes.
*
* @param source the file to process
* @param contextPathPos the position on the include path where the containing file was found,
* or <code>-1</code> for top-level inclusions
* @param contextKind the kind how the containing file was included, or null for top-level
* inclusions
* @param visited the set to receive the files that are transitively included by {@code source}
* @param pathToLegalOutputArtifact map to look up legal output artifact by path
* @param actionExecutionMetadata owning action
* @param actionExecutionContext Services in the scope of the action, like the stream to which
* @param visitedInclusions the set of all processed inclusions, to avoid processing duplicate
* inclusions.
*/
private void process(
final Artifact source,
int contextPathPos,
Kind contextKind,
Set<Artifact> visited,
Map<PathFragment, Artifact> pathToLegalOutputArtifact,
final ActionExecutionMetadata actionExecutionMetadata,
final ActionExecutionContext actionExecutionContext,
Set<ArtifactWithInclusionContext> visitedInclusions,
final Artifact grepIncludes)
throws IOException, ExecException, InterruptedException {
checkForInterrupt("processing", source);
Collection<Inclusion> inclusions = null;
try {
inclusions =
fileParseCache.computeIfAbsent(
source,
file -> {
try {
return parser.extractInclusions(
file,
actionExecutionMetadata,
actionExecutionContext,
grepIncludes,
spawnIncludeScannerSupplier.get(),
isRealOutputFile(source.getExecPath()));
} catch (IOException e) {
throw new IORuntimeException(e);
} catch (ExecException e) {
throw new ExecRuntimeException(e);
} catch (InterruptedException e) {
throw new InterruptedRuntimeException(e);
}
});
} catch (IORuntimeException e) {
throw e.getCauseIOException();
} catch (ExecRuntimeException e) {
throw e.getRealCause();
} catch (InterruptedRuntimeException e) {
throw e.getRealCause();
}
Preconditions.checkNotNull(inclusions, source);
// Shuffle the inclusions to get better parallelism. See b/62200470.
List<Inclusion> shuffledInclusions = new ArrayList<>(inclusions);
Collections.shuffle(shuffledInclusions, CONSTANT_SEED_RANDOM);
// For each inclusion: get or locate its target file & recursively process
IncludeScannerHelper helper =
new IncludeScannerHelper(includePaths, quoteIncludePaths, source);
for (Inclusion inclusion : shuffledInclusions) {
findAndProcess(
helper.createInclusionWithContext(inclusion, contextPathPos, contextKind),
source,
visited,
pathToLegalOutputArtifact,
actionExecutionMetadata,
actionExecutionContext,
visitedInclusions,
grepIncludes);
}
}
/**
* Same as {@link #process}, but executes asynchronously if the #include lines of {@code source}
* haven't been extracted yet. For sources that have already been extracted, just continue
* walking them in the current thread. The overhead of scheduling this on other threads is
* larger than the gain in concurrency. The only really slow operation is the (possibly remote)
* extraction of includes.
*/
private void processAsyncIfNotExtracted(
final Artifact source,
int contextPathPos,
Kind contextKind,
Set<Artifact> visited,
Map<PathFragment, Artifact> pathToLegalOutputArtifact,
final ActionExecutionMetadata actionExecutionMetadata,
final ActionExecutionContext actionExecutionContext,
Set<ArtifactWithInclusionContext> visitedInclusions,
final Artifact grepIncludes)
throws IOException, ExecException, InterruptedException {
Collection<Inclusion> cacheResult = fileParseCache.get(source);
if (cacheResult != null) {
process(
source,
contextPathPos,
contextKind,
visited,
pathToLegalOutputArtifact,
actionExecutionMetadata,
actionExecutionContext,
visitedInclusions,
grepIncludes);
} else {
super.execute(() -> {
try {
process(
source,
contextPathPos,
contextKind,
visited,
pathToLegalOutputArtifact,
actionExecutionMetadata,
actionExecutionContext,
visitedInclusions,
grepIncludes);
} catch (IOException e) {
throw new IORuntimeException(e);
} catch (ExecException e) {
throw new ExecRuntimeException(e);
} catch (InterruptedException e) {
throw new InterruptedRuntimeException(e);
}
});
}
}
/** Visits an inclusion starting from a source file. */
private void findAndProcess(
InclusionWithContext inclusion,
Artifact source,
Set<Artifact> visited,
Map<PathFragment, Artifact> pathToLegalOutputArtifact,
ActionExecutionMetadata actionExecutionMetadata,
ActionExecutionContext actionExecutionContext,
Set<ArtifactWithInclusionContext> visitedInclusions,
Artifact grepIncludes)
throws IOException, ExecException, InterruptedException {
// Try to find the included file relative to the file that contains the inclusion. Relative
// inclusions are handled like the first entry on the quote include path
Artifact includeFile =
locateRelative(inclusion.getInclusion(), pathToLegalOutputArtifact, source);
int contextPathPos = 0;
Kind contextKind = null;
checkForInterrupt("visiting", source);
// If nothing has been found, get an inclusion from the cache. This will automatically search
// on the include paths and populate the cache if necessary.
if (includeFile == null) {
LocateOnPathResult result = inclusionCache.lookup(inclusion, pathToLegalOutputArtifact);
includeFile = result.path;
contextPathPos = result.includePosition;
contextKind = inclusion.getContextKind();
}
// Recursively process the found file (if not yet done).
if (includeFile != null
&& !isIllegalOutputFile(includeFile.getExecPath(), pathToLegalOutputArtifact.keySet())
&& visitedInclusions.add(
new ArtifactWithInclusionContext(includeFile, contextKind, contextPathPos))) {
visited.add(includeFile);
if (modularHeaders.contains(includeFile)) {
return;
}
processAsyncIfNotExtracted(
includeFile,
contextPathPos,
contextKind,
visited,
pathToLegalOutputArtifact,
actionExecutionMetadata,
actionExecutionContext,
visitedInclusions,
grepIncludes);
}
}
/**
* Processes a given list of includes for a given base file and populates the provided set with
* the visited includes
*
* @param source the source file used as a reference for finding includes
* @param includes the list of -include option strings to locate and process
* @param visited the set of files that are transitively included by {@code includes} to
* populate
* @param pathToLegalOutputArtifact map to look up legal output artifact by path
* @param actionExecutionContext Services in the scope of the action, like the stream to which
* @param visitedInclusions the set of all processed inclusions, to avoid processing duplicate
* inclusions.
*/
private void processCmdlineIncludes(
Artifact source,
List<String> includes,
Artifact grepIncludes,
Set<Artifact> visited,
Map<PathFragment, Artifact> pathToLegalOutputArtifact,
ActionExecutionMetadata actionExecutionMetadata,
ActionExecutionContext actionExecutionContext,
Set<ArtifactWithInclusionContext> visitedInclusions)
throws IOException, ExecException, InterruptedException {
for (String incl : includes) {
InclusionWithContext inclusion = new InclusionWithContext(incl, Kind.QUOTE);
findAndProcess(
inclusion,
source,
visited,
pathToLegalOutputArtifact,
actionExecutionMetadata,
actionExecutionContext,
visitedInclusions,
grepIncludes);
}
}
/**
* Processes a bunch sources asynchronously and adds them and their included files to the
* provided set.
*
* @param sources the files to process and add to the set
* @param visited the set to receive the files that are transitively included by {@code sources}
* @param pathToLegalOutputArtifact map to look up legal output artifact by path
* @param actionExecutionContext Services in the scope of the action, like the stream to which
* @param visitedInclusions the set of all processed inclusions, to avoid processing duplicate
* inclusions.
*/
private void processBulkAsync(
Collection<Artifact> sources,
final Set<Artifact> visited,
final Map<PathFragment, Artifact> pathToLegalOutputArtifact,
final ActionExecutionMetadata actionExecutionMetadata,
final ActionExecutionContext actionExecutionContext,
final Set<ArtifactWithInclusionContext> visitedInclusions,
Artifact grepIncludes)
throws IOException, ExecException, InterruptedException {
for (final Artifact source : sources) {
// TODO(djasper): This looks suspicious. We should only stop based on visitedInclusions.
if (!visited.add(source)) {
continue;
}
processAsyncIfNotExtracted(
source,
/*contextPathPos=*/ -1,
/*contextKind=*/ null,
visited,
pathToLegalOutputArtifact,
actionExecutionMetadata,
actionExecutionContext,
visitedInclusions,
grepIncludes);
}
}
private void processFileLevelHintsAsync(
final Hints hints,
final Artifact include,
final Set<Artifact> alsoVisited,
final Map<PathFragment, Artifact> pathToLegalOutputArtifact,
final ActionExecutionMetadata actionExecutionMetadata,
final ActionExecutionContext actionExecutionContext,
final Set<ArtifactWithInclusionContext> visitedInclusions,
Artifact grepIncludes) {
Collection<Artifact> sources = hints.getFileLevelHintedInclusionsLegacy(include);
// Early-out if there's nothing to do to avoid enqueuing a closure
if (sources.isEmpty()) {
return;
}
super.execute(() -> {
try {
processBulkAsync(
sources,
alsoVisited,
pathToLegalOutputArtifact,
actionExecutionMetadata,
actionExecutionContext,
visitedInclusions,
grepIncludes);
} catch (IOException e) {
throw new IORuntimeException(e);
} catch (ExecException e) {
throw new ExecRuntimeException(e);
} catch (InterruptedException e) {
throw new InterruptedRuntimeException(e);
}
});
}
}
private static class ExecRuntimeException extends RuntimeException {
private final ExecException cause;
public ExecRuntimeException(ExecException e) {
super(e);
this.cause = e;
}
public ExecException getRealCause() {
return cause;
}
}
private static class InterruptedRuntimeException extends RuntimeException {
private final InterruptedException cause;
public InterruptedRuntimeException(InterruptedException e) {
super(e);
this.cause = e;
}
public InterruptedException getRealCause() {
return cause;
}
}
}