| // Copyright 2018 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| package com.google.devtools.build.lib.includescanning; |
| |
| import com.google.common.base.Preconditions; |
| import com.google.common.base.Supplier; |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableSet; |
| import com.google.common.collect.Sets; |
| import com.google.devtools.build.lib.actions.ActionExecutionContext; |
| import com.google.devtools.build.lib.actions.ActionExecutionMetadata; |
| import com.google.devtools.build.lib.actions.Artifact; |
| import com.google.devtools.build.lib.actions.ArtifactFactory; |
| import com.google.devtools.build.lib.actions.ArtifactRoot; |
| import com.google.devtools.build.lib.actions.ExecException; |
| import com.google.devtools.build.lib.actions.MissingDepException; |
| import com.google.devtools.build.lib.analysis.BlazeDirectories; |
| import com.google.devtools.build.lib.cmdline.RepositoryName; |
| import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor; |
| import com.google.devtools.build.lib.concurrent.ErrorClassifier; |
| import com.google.devtools.build.lib.concurrent.ThreadSafety; |
| import com.google.devtools.build.lib.includescanning.IncludeParser.Hints; |
| import com.google.devtools.build.lib.includescanning.IncludeParser.Inclusion; |
| import com.google.devtools.build.lib.includescanning.IncludeParser.Inclusion.Kind; |
| import com.google.devtools.build.lib.rules.cpp.IncludeScanner; |
| import com.google.devtools.build.lib.vfs.IORuntimeException; |
| import com.google.devtools.build.lib.vfs.Path; |
| import com.google.devtools.build.lib.vfs.PathFragment; |
| import com.google.devtools.build.lib.vfs.Root; |
| import com.google.devtools.build.skyframe.SkyFunction; |
| import java.io.IOException; |
| import java.util.ArrayList; |
| import java.util.Collection; |
| import java.util.Collections; |
| import java.util.Iterator; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Objects; |
| import java.util.Random; |
| import java.util.Set; |
| import java.util.concurrent.ConcurrentHashMap; |
| import java.util.concurrent.ConcurrentMap; |
| import java.util.concurrent.ExecutorService; |
| |
| /** |
| * C include scanner. Quickly scans C/C++ source files to determine the bounding set of transitively |
| * referenced include files. |
| * |
| * <p>Maintains caches for parses and search-matches for performance. |
| * |
| * <pre> |
| * TODO(bazel-team): (2009) Currently does not evaluate preprocessor symbols, so computed includes |
| * are ignored. |
| * TODO(bazel-team): (2009) Does not handle multiline block comments preceding or around an #include |
| * </pre> |
| */ |
| public class LegacyIncludeScanner implements IncludeScanner { |
| |
| private static final class ArtifactWithInclusionContext { |
| private final Artifact artifact; |
| private final Kind contextKind; |
| private final int contextPathPos; |
| |
| private ArtifactWithInclusionContext(Artifact artifact, Kind contextKind, int contextPathPos) { |
| this.artifact = artifact; |
| this.contextKind = contextKind; |
| this.contextPathPos = contextPathPos; |
| } |
| |
| @Override |
| public int hashCode() { |
| return contextPathPos + 37 * Objects.hash(contextKind, artifact); |
| } |
| |
| @Override |
| public boolean equals(Object obj) { |
| if (this == obj) { |
| return true; |
| } |
| if (!(obj instanceof ArtifactWithInclusionContext)) { |
| return false; |
| } |
| ArtifactWithInclusionContext that = (ArtifactWithInclusionContext) obj; |
| return this.contextKind == that.contextKind |
| && this.contextPathPos == that.contextPathPos |
| && this.artifact.equals(that.artifact); |
| } |
| } |
| |
| /** |
| * A cache of inclusion lookups, taking care to avoid spurious caching related |
| * to generated headers / source files. |
| */ |
| @ThreadSafety.ThreadSafe |
| private class InclusionCache { |
| private final ConcurrentMap<InclusionWithContext, LocateOnPathResult> cache = |
| new ConcurrentHashMap<>(); |
| |
| /** |
| * Locates an included file along the search paths. The result is cacheable. |
| * |
| * @param inclusion the inclusion to locate |
| * @param pathToLegalOutputArtifact generated files which may be reached during scanning |
| * @param onlyCheckGenerated if true, only search for generated output files |
| * @return a tuple of the found file, the position of the respective include path entry on the |
| * search path (or null if no matching file was found), and whether the scan touched illegal |
| * output files |
| */ |
| private LocateOnPathResult locateOnPaths( |
| InclusionWithContext inclusion, |
| Map<PathFragment, Artifact> pathToLegalOutputArtifact, |
| boolean onlyCheckGenerated) { |
| PathFragment name = inclusion.getInclusion().pathFragment; |
| |
| // For #include_next directives we start searching on the include path where |
| // we found the previous inclusion. |
| int searchStart = inclusion.getInclusion().kind.isNext() ? inclusion.getContextPathPos() : 0; |
| |
| // Search the header on the remaining paths. |
| List<PathFragment> paths = |
| inclusion.getContextKind() == Kind.QUOTE ? quoteIncludePaths : includePaths; |
| boolean viewedIllegalOutput = false; |
| for (int i = searchStart; i < paths.size(); ++i) { |
| PathFragment fileFragment = paths.get(i).getRelative(name); |
| if (fileFragment.containsUplevelReferences()) { |
| // TODO(janakr): This branch shouldn't be necessary: we should be able to filter such |
| // inclusions out unconditionally. |
| // Deal with fragments that escape the execroot. They most likely come right back in. |
| Path execRootRelativePath = execRoot.getRelative(fileFragment); |
| if (execRootRelativePath.startsWith(execRoot)) { |
| // Common case: transform #include "../execroot/foo.h" into #include "foo.h" |
| fileFragment = execRootRelativePath.relativeTo(execRoot); |
| } else { |
| // Ugh: successfully escaped the exec root. It's their funeral. |
| fileFragment = execRootRelativePath.asFragment(); |
| } |
| // This can happen when we are processing Windows paths with backslashes on Unix, |
| // since we do not do any #ifdef processing. |
| // We can safely discard these here. |
| if (fileFragment.containsUplevelReferences()) { |
| continue; |
| } |
| } |
| if (onlyCheckGenerated && !isRealOutputFile(fileFragment)) { |
| continue; |
| } |
| viewedIllegalOutput = |
| viewedIllegalOutput |
| || isIllegalOutputFile(fileFragment, pathToLegalOutputArtifact.keySet()); |
| boolean isOutputDirectory = fileFragment.startsWith(outputPathFragment); |
| if (!isFile(fileFragment, name, !isOutputDirectory, pathToLegalOutputArtifact.keySet())) { |
| continue; |
| } |
| Artifact artifact; |
| if (isOutputDirectory) { |
| // May be a normal output file or an inc_library header. |
| artifact = pathToLegalOutputArtifact.get(fileFragment); |
| if (artifact == null) { |
| // This happens if an included file exists in a cc_inc_library's output directory, |
| // but is not an output of the cc_inc_library. This can happen if, for instance, the |
| // definition of the cc_inc_library is changed to output different files, but the |
| // source file's includes don't change. |
| // Often, such an include is conditional, and so failing to find it here will not |
| // lead to problems. If this include is actually needed for compilation, then we will |
| // emit a somewhat unhelpful error message of a missing file, rather than the more |
| // helpful one of an illegal include, but it's hard to emit the illegal include |
| // message consistently, and this is a rare occurrence in any case. |
| return LocateOnPathResult.createNotFound(viewedIllegalOutput); |
| } |
| } else if (!fileFragment.isAbsolute()) { |
| artifact = artifactFactory.resolveSourceArtifact(fileFragment, RepositoryName.MAIN); |
| if (artifact == null) { |
| // There was a real file, but we couldn't resolve it, probably because it belonged to |
| // a package that wasn't actually loaded this build, so user cannot refer to files in |
| // that package. |
| continue; |
| } |
| } else { |
| // This file is given with an absolute path. We will error out after transitive scanning |
| // of the top-level source is finished unless this corresponds to a built-in include |
| // directory, and will ignore this artifact in any case, but track it here so that its |
| // includes can be processed. |
| artifact = artifactFactory.getSourceArtifact(fileFragment, absoluteRoot); |
| } |
| // +1 to account for the virtual entry for relative includes. |
| return LocateOnPathResult.create(artifact, i + 1, viewedIllegalOutput); |
| } |
| |
| // Not found. |
| return LocateOnPathResult.createNotFound(viewedIllegalOutput); |
| } |
| |
| /** |
| * Locates an included file along the search paths. |
| * |
| * @param inclusion the inclusion to locate |
| * @param pathToLegalOutputArtifact generated files which may be reached during scanning |
| * @return a LocateOnPathResult |
| */ |
| public LocateOnPathResult lookup( |
| InclusionWithContext inclusion, Map<PathFragment, Artifact> pathToLegalOutputArtifact) { |
| LocateOnPathResult result = |
| cache.computeIfAbsent( |
| inclusion, key -> locateOnPaths(key, pathToLegalOutputArtifact, false)); |
| // It is not safe to cache lookups which viewed illegal output files. Their nonexistence do |
| // not imply nonexistence for actions using the same include scanner, but executed later on. |
| // See bug 2097998. For performance reasons, take a small shortcut: only avoid caching when |
| // the path lookup result from locateOnPaths() is empty. |
| if (result.path != null || !result.viewedIllegalOutputFile) { |
| return result; |
| } |
| |
| result = locateOnPaths(inclusion, pathToLegalOutputArtifact, true); |
| if (result.path != null || !result.viewedIllegalOutputFile) { |
| // In this case, the result is now cachable either because a file has been found or |
| // because there are no more illegal output files. This is rare in practice. Avoid |
| // creating a future and modifying the cache in the common case. |
| cache.put(inclusion, result); |
| } |
| return result; |
| } |
| } |
| |
| private static class LocateOnPathResult { |
| private static final LocateOnPathResult NOT_FOUND_VIEWED_ILLEGAL = |
| new LocateOnPathResult(null, -1, true); |
| private static final LocateOnPathResult NOT_FOUND_NO_VIEWED_ILLEGAL = |
| new LocateOnPathResult(null, -1, false); |
| private final Artifact path; |
| private final int includePosition; |
| private final boolean viewedIllegalOutputFile; |
| |
| private LocateOnPathResult( |
| Artifact path, int includePosition, boolean viewedIllegalOutputFile) { |
| this.path = path; |
| this.includePosition = includePosition; |
| this.viewedIllegalOutputFile = viewedIllegalOutputFile; |
| } |
| |
| static LocateOnPathResult create( |
| Artifact path, int includePosition, boolean viewedIllegalOutputFile) { |
| return new LocateOnPathResult( |
| Preconditions.checkNotNull(path), includePosition, viewedIllegalOutputFile); |
| } |
| |
| static LocateOnPathResult createNotFound(boolean viewedIllegalOutputFile) { |
| return viewedIllegalOutputFile ? NOT_FOUND_VIEWED_ILLEGAL : NOT_FOUND_NO_VIEWED_ILLEGAL; |
| } |
| } |
| |
| private final Path execRoot; |
| |
| private final ArtifactFactory artifactFactory; |
| private final Supplier<SpawnIncludeScanner> spawnIncludeScannerSupplier; |
| |
| /** |
| * Externally-scoped cache of file path => parsed inclusion set mappings. Saves us from having to |
| * parse files more than once, and can be shared by scanners with different search paths. |
| */ |
| private final ConcurrentMap<Artifact, Collection<Inclusion>> fileParseCache; |
| |
| private final IncludeParser parser; |
| |
| /** |
| * Search path for searching for all quoted "xyz.h" includes, composed of all the -iquote, -I and |
| * -isystem paths (in this order). |
| */ |
| private final ImmutableList<PathFragment> quoteIncludePaths; |
| |
| /** |
| * Search path for searching for all includes, composed of all the -I and -isystem paths (in this |
| * order). |
| */ |
| private final List<PathFragment> includePaths; |
| |
| private final PathFragment includeRootFragment; |
| private final PathFragment outputPathFragment; |
| private final Root absoluteRoot; |
| |
| /** |
| * Scanner-scoped cache of inclusions with their resolved files and include path entries. This |
| * cache is specific to a given pair of search paths, and is thus scanner-local. |
| * |
| * <p>Each inclusion (name+type+context) is associated with its resolved file here, thus saving |
| * redundant path searches. The second entry of the pair is the include path entry on which the |
| * file was found. |
| */ |
| private final InclusionCache inclusionCache; |
| |
| private final PathExistenceCache pathCache; |
| |
| private final ExecutorService includePool; |
| |
| // We are using this Random just for shuffling, so keep the order deterministic by hardcoding |
| // the seed. |
| private static final Random CONSTANT_SEED_RANDOM = new Random(88); |
| |
| /** |
| * Constructs a new IncludeScanner |
| * |
| * @param cache externally scoped cache of file-path to inclusion-set mappings |
| * @param pathCache include path existence cache |
| * @param quoteIncludePaths the list of quote search path dirs (-iquote) |
| * @param includePaths the list of all other search path dirs (-I and -isystem) |
| */ |
| LegacyIncludeScanner( |
| IncludeParser parser, |
| ExecutorService includePool, |
| ConcurrentMap<Artifact, Collection<Inclusion>> cache, |
| PathExistenceCache pathCache, |
| List<PathFragment> quoteIncludePaths, |
| List<PathFragment> includePaths, |
| Path outputPath, |
| Path execRoot, |
| ArtifactFactory artifactFactory, |
| Supplier<SpawnIncludeScanner> spawnIncludeScannerSupplier) { |
| this.parser = parser; |
| this.includePool = includePool; |
| this.fileParseCache = cache; |
| this.pathCache = pathCache; |
| this.artifactFactory = Preconditions.checkNotNull(artifactFactory); |
| this.spawnIncludeScannerSupplier = spawnIncludeScannerSupplier; |
| this.quoteIncludePaths = ImmutableList.<PathFragment>builder() |
| .addAll(quoteIncludePaths) |
| .addAll(includePaths) |
| .build(); |
| this.includePaths = ImmutableList.copyOf(includePaths); |
| this.inclusionCache = new InclusionCache(); |
| this.execRoot = execRoot; |
| this.outputPathFragment = outputPath.relativeTo(execRoot); |
| this.includeRootFragment = |
| outputPathFragment.getRelative(BlazeDirectories.RELATIVE_INCLUDE_DIR); |
| this.absoluteRoot = Root.absoluteRoot(execRoot.getFileSystem()); |
| } |
| |
| /** |
| * Locates an included file relative to the including file. The result is not cacheable. |
| * |
| * @param inclusion the inclusion to locate |
| * @param includer the including file |
| * @return the resolved Path, or null if no file could be found |
| */ |
| private Artifact locateRelative( |
| Inclusion inclusion, Map<PathFragment, Artifact> legalOutputFiles, Artifact includer) { |
| if (inclusion.kind != Kind.QUOTE) { |
| return null; |
| } |
| PathFragment name = inclusion.pathFragment; |
| PathFragment parentDirectory = includer.getRootRelativePath().getParentDirectory(); |
| PathFragment rootRelativePath = parentDirectory.getRelative(name); |
| if (rootRelativePath.containsUplevelReferences()) { |
| // An include cannot break out of a (package path) root via a relative inclusion. It should |
| // also not break out of the root and then come back into it -- who knows what hardcoded |
| // directory names there are in it. |
| return null; |
| } |
| PathFragment execPath = includer.getExecPath().getParentDirectory().getRelative(name); |
| if (!isFile(execPath, name, includer.isSourceArtifact(), legalOutputFiles.keySet())) { |
| return null; |
| } |
| if (legalOutputFiles.containsKey(execPath)) { |
| return legalOutputFiles.get(execPath); |
| } |
| ArtifactRoot root = includer.getRoot(); |
| Artifact sourceArtifact = |
| artifactFactory.resolveSourceArtifactWithAncestor( |
| name, parentDirectory, root, RepositoryName.MAIN); |
| if (sourceArtifact == null) { |
| // If the name had up-level references, this path may not be under any package. Otherwise, |
| // we must have gotten an artifact, since it should be under the same package as the |
| // including artifact. |
| Preconditions.checkState( |
| name.containsUplevelReferences(), |
| "%s %s %s %s", |
| name, |
| parentDirectory, |
| rootRelativePath, |
| root); |
| } |
| return sourceArtifact; |
| } |
| |
| /** Returns whether the given path exists in the filesystem. */ |
| private boolean isFile( |
| PathFragment execPath, PathFragment includeAsWritten, boolean isSource, |
| Collection<PathFragment> legalOutputFiles) { |
| if (isRealOutputFile(execPath)) { |
| return legalOutputFiles.contains(execPath); |
| } |
| // TODO(djasper): This code path cannot be hit with isSource being false. Verify and add |
| // Preconditions check. |
| if (isSource && !execPath.isAbsolute() && execPath.endsWith(includeAsWritten)) { |
| // Verify that the directory of execPath exists as an optimization. Most includes are relative |
| // to the workspace and we'd like to avoid stat'ing every such include relative to every |
| // include path. If testing whether "a/b/c.h" is a file beneath the include path "e/f/", |
| // verify that "e/f/a" and "e/f/a/b" are valid directories (and cache the result). |
| int execPathSegments = execPath.segmentCount(); |
| int nameSegments = includeAsWritten.segmentCount(); |
| for (int i = execPathSegments - nameSegments + 1; i < execPathSegments; i++) { |
| if (!pathCache.directoryExists(execPath.subFragment(0, i))) { |
| return false; |
| } |
| } |
| } |
| return pathCache.fileExists(execPath, isSource); |
| } |
| |
| @Override |
| public void process( |
| Artifact mainSource, |
| Collection<Artifact> sources, |
| IncludeScanningHeaderData includeScanningHeaderData, |
| List<String> cmdlineIncludes, |
| Set<Artifact> includes, |
| ActionExecutionMetadata actionExecutionMetadata, |
| ActionExecutionContext actionExecutionContext, |
| Artifact grepIncludes) |
| throws IOException, ExecException, InterruptedException { |
| try { |
| // Because our IncludeVisitor can dynamically switch to in-thread execution, we could get |
| // unlucky and run one of the async calls on the main thread. Catch and rethrow the runtime |
| // exceptions as the appropriate corresponding checked exception. |
| processInternal( |
| mainSource, |
| sources, |
| includeScanningHeaderData, |
| cmdlineIncludes, |
| includes, |
| actionExecutionMetadata, |
| actionExecutionContext, |
| grepIncludes); |
| } catch (IORuntimeException e) { |
| throw e.getCauseIOException(); |
| } catch (ExecRuntimeException e) { |
| throw e.getRealCause(); |
| } catch (InterruptedRuntimeException e) { |
| throw e.getRealCause(); |
| } |
| } |
| |
| private void processInternal( |
| Artifact mainSource, |
| Collection<Artifact> sources, |
| IncludeScanningHeaderData includeScanningHeaderData, |
| List<String> cmdlineIncludes, |
| Set<Artifact> includes, |
| ActionExecutionMetadata actionExecutionMetadata, |
| ActionExecutionContext actionExecutionContext, |
| Artifact grepIncludes) |
| throws IOException, ExecException, InterruptedException { |
| Preconditions.checkArgument(mainSource == null || sources.contains(mainSource), |
| "The main source '%s' is not part of '%s'", mainSource, sources); |
| ImmutableSet.Builder<Artifact> pathHints = null; |
| SkyFunction.Environment env = actionExecutionContext.getEnvironmentForDiscoveringInputs(); |
| if (parser.getHints() != null) { |
| pathHints = ImmutableSet.builderWithExpectedSize(quoteIncludePaths.size()); |
| Collection<Artifact> artifacts = |
| parser.getHints().getPathLevelHintedInclusions(quoteIncludePaths, env); |
| if (!env.valuesMissing()) { |
| pathHints.addAll(Preconditions.checkNotNull(artifacts, quoteIncludePaths)); |
| } |
| } |
| if (env.valuesMissing()) { |
| throw new MissingDepException(); |
| } |
| |
| Set<ArtifactWithInclusionContext> visitedInclusions = Sets.newConcurrentHashSet(); |
| |
| IncludeVisitor visitor = new IncludeVisitor(includeScanningHeaderData.getModularHeaders()); |
| |
| try { |
| // Process cmd line includes, if specified. |
| if (mainSource != null && !cmdlineIncludes.isEmpty()) { |
| visitor.processCmdlineIncludes( |
| mainSource, |
| cmdlineIncludes, |
| grepIncludes, |
| includes, |
| includeScanningHeaderData.getPathToLegalOutputArtifact(), |
| actionExecutionMetadata, |
| actionExecutionContext, |
| visitedInclusions); |
| visitor.sync(); |
| } |
| |
| visitor.processBulkAsync( |
| sources, |
| includes, |
| includeScanningHeaderData.getPathToLegalOutputArtifact(), |
| actionExecutionMetadata, |
| actionExecutionContext, |
| visitedInclusions, |
| grepIncludes); |
| visitor.sync(); |
| |
| // Process include hints |
| // TODO(ulfjack): Make this code go away. Use the new hinted inclusions instead. |
| Hints hints = parser.getHints(); |
| if (hints != null) { |
| // Follow "path" hints. |
| visitor.processBulkAsync( |
| pathHints.build(), |
| includes, |
| includeScanningHeaderData.getPathToLegalOutputArtifact(), |
| actionExecutionMetadata, |
| actionExecutionContext, |
| visitedInclusions, |
| grepIncludes); |
| // Follow "file" hints for the primary sources. |
| for (Artifact source : sources) { |
| visitor.processFileLevelHintsAsync( |
| hints, |
| source, |
| includes, |
| includeScanningHeaderData.getPathToLegalOutputArtifact(), |
| actionExecutionMetadata, |
| actionExecutionContext, |
| visitedInclusions, |
| grepIncludes); |
| } |
| visitor.sync(); |
| |
| // Follow "file" hints for all included headers, transitively. |
| Set<Artifact> frontier = includes; |
| while (!frontier.isEmpty()) { |
| Set<Artifact> adjacent = Sets.newConcurrentHashSet(); |
| for (Artifact include : frontier) { |
| visitor.processFileLevelHintsAsync( |
| hints, |
| include, |
| adjacent, |
| includeScanningHeaderData.getPathToLegalOutputArtifact(), |
| actionExecutionMetadata, |
| actionExecutionContext, |
| visitedInclusions, |
| grepIncludes); |
| } |
| visitor.sync(); |
| // Keep novel nodes as the next frontier. |
| for (Iterator<Artifact> iter = adjacent.iterator(); iter.hasNext(); ) { |
| if (!includes.add(iter.next())) { |
| iter.remove(); |
| } |
| } |
| frontier = adjacent; |
| } |
| } |
| } catch (IOException | InterruptedException | ExecException | MissingDepException e) { |
| // Careful: Do not leak visitation threads if we have an exception in the initial thread. |
| visitor.sync(); |
| throw e; |
| } |
| } |
| |
| private static void checkForInterrupt(String operation, Object source) |
| throws InterruptedException { |
| // We require passing in the operation and the source Path / Artifact to avoid intermediate |
| // String operations. The include scanner is performance critical and this showed up in a |
| // profiler. |
| if (Thread.currentThread().isInterrupted()) { |
| throw new InterruptedException( |
| "Include scanning interrupted while " + operation + " " + source); |
| } |
| } |
| |
| private boolean isIllegalOutputFile( |
| PathFragment includeFile, Collection<PathFragment> legalOutputFiles) { |
| return isRealOutputFile(includeFile) && !legalOutputFiles.contains(includeFile); |
| } |
| |
| private boolean isRealOutputFile(PathFragment path) { |
| return path.startsWith(outputPathFragment) && !isIncPath(path); |
| } |
| |
| private boolean isIncPath(PathFragment path) { |
| // See CreateIncSymlinkAction and where it's used: The symlink trees |
| // are always rooted at locations that fit the logic here. |
| return path.startsWith(includeRootFragment) && !path.equals(includeRootFragment); |
| } |
| |
| /** |
| * Implements a potentially parallel traversal over source files using a |
| * thread pool shared across different IncludeScanner instances. |
| */ |
| private class IncludeVisitor extends AbstractQueueVisitor { |
| /** The set of headers known to be part of a C++ module. Scanning can stop here. */ |
| private Set<Artifact> modularHeaders; |
| |
| public IncludeVisitor(Set<Artifact> modularHeaders) { |
| super( |
| includePool, |
| /*shutdownOnCompletion=*/ false, |
| /*failFastOnException=*/ true, |
| ErrorClassifier.DEFAULT); |
| this.modularHeaders = modularHeaders; |
| } |
| |
| /** |
| * Block for the completion of all outstanding visitations. |
| */ |
| public void sync() throws IOException, ExecException, InterruptedException { |
| try { |
| super.awaitQuiescence(true); |
| } catch (InterruptedException e) { |
| throw new InterruptedException("Interrupted during include visitation"); |
| } catch (IORuntimeException e) { |
| throw e.getCauseIOException(); |
| } catch (ExecRuntimeException e) { |
| throw e.getRealCause(); |
| } catch (InterruptedRuntimeException e) { |
| throw e.getRealCause(); |
| } |
| } |
| |
| /** |
| * Processes a given file for includes and populates the provided set with the visited includes. |
| * |
| * @param source the file to process |
| * @param contextPathPos the position on the include path where the containing file was found, |
| * or <code>-1</code> for top-level inclusions |
| * @param contextKind the kind how the containing file was included, or null for top-level |
| * inclusions |
| * @param visited the set to receive the files that are transitively included by {@code source} |
| * @param pathToLegalOutputArtifact map to look up legal output artifact by path |
| * @param actionExecutionMetadata owning action |
| * @param actionExecutionContext Services in the scope of the action, like the stream to which |
| * @param visitedInclusions the set of all processed inclusions, to avoid processing duplicate |
| * inclusions. |
| */ |
| private void process( |
| final Artifact source, |
| int contextPathPos, |
| Kind contextKind, |
| Set<Artifact> visited, |
| Map<PathFragment, Artifact> pathToLegalOutputArtifact, |
| final ActionExecutionMetadata actionExecutionMetadata, |
| final ActionExecutionContext actionExecutionContext, |
| Set<ArtifactWithInclusionContext> visitedInclusions, |
| final Artifact grepIncludes) |
| throws IOException, ExecException, InterruptedException { |
| checkForInterrupt("processing", source); |
| |
| Collection<Inclusion> inclusions = null; |
| try { |
| inclusions = |
| fileParseCache.computeIfAbsent( |
| source, |
| file -> { |
| try { |
| return parser.extractInclusions( |
| file, |
| actionExecutionMetadata, |
| actionExecutionContext, |
| grepIncludes, |
| spawnIncludeScannerSupplier.get(), |
| isRealOutputFile(source.getExecPath())); |
| } catch (IOException e) { |
| throw new IORuntimeException(e); |
| } catch (ExecException e) { |
| throw new ExecRuntimeException(e); |
| } catch (InterruptedException e) { |
| throw new InterruptedRuntimeException(e); |
| } |
| }); |
| } catch (IORuntimeException e) { |
| throw e.getCauseIOException(); |
| } catch (ExecRuntimeException e) { |
| throw e.getRealCause(); |
| } catch (InterruptedRuntimeException e) { |
| throw e.getRealCause(); |
| } |
| Preconditions.checkNotNull(inclusions, source); |
| |
| // Shuffle the inclusions to get better parallelism. See b/62200470. |
| List<Inclusion> shuffledInclusions = new ArrayList<>(inclusions); |
| Collections.shuffle(shuffledInclusions, CONSTANT_SEED_RANDOM); |
| |
| // For each inclusion: get or locate its target file & recursively process |
| IncludeScannerHelper helper = |
| new IncludeScannerHelper(includePaths, quoteIncludePaths, source); |
| for (Inclusion inclusion : shuffledInclusions) { |
| findAndProcess( |
| helper.createInclusionWithContext(inclusion, contextPathPos, contextKind), |
| source, |
| visited, |
| pathToLegalOutputArtifact, |
| actionExecutionMetadata, |
| actionExecutionContext, |
| visitedInclusions, |
| grepIncludes); |
| } |
| } |
| |
| /** |
| * Same as {@link #process}, but executes asynchronously if the #include lines of {@code source} |
| * haven't been extracted yet. For sources that have already been extracted, just continue |
| * walking them in the current thread. The overhead of scheduling this on other threads is |
| * larger than the gain in concurrency. The only really slow operation is the (possibly remote) |
| * extraction of includes. |
| */ |
| private void processAsyncIfNotExtracted( |
| final Artifact source, |
| int contextPathPos, |
| Kind contextKind, |
| Set<Artifact> visited, |
| Map<PathFragment, Artifact> pathToLegalOutputArtifact, |
| final ActionExecutionMetadata actionExecutionMetadata, |
| final ActionExecutionContext actionExecutionContext, |
| Set<ArtifactWithInclusionContext> visitedInclusions, |
| final Artifact grepIncludes) |
| throws IOException, ExecException, InterruptedException { |
| Collection<Inclusion> cacheResult = fileParseCache.get(source); |
| if (cacheResult != null) { |
| process( |
| source, |
| contextPathPos, |
| contextKind, |
| visited, |
| pathToLegalOutputArtifact, |
| actionExecutionMetadata, |
| actionExecutionContext, |
| visitedInclusions, |
| grepIncludes); |
| } else { |
| super.execute(() -> { |
| try { |
| process( |
| source, |
| contextPathPos, |
| contextKind, |
| visited, |
| pathToLegalOutputArtifact, |
| actionExecutionMetadata, |
| actionExecutionContext, |
| visitedInclusions, |
| grepIncludes); |
| } catch (IOException e) { |
| throw new IORuntimeException(e); |
| } catch (ExecException e) { |
| throw new ExecRuntimeException(e); |
| } catch (InterruptedException e) { |
| throw new InterruptedRuntimeException(e); |
| } |
| }); |
| } |
| } |
| |
| /** Visits an inclusion starting from a source file. */ |
| private void findAndProcess( |
| InclusionWithContext inclusion, |
| Artifact source, |
| Set<Artifact> visited, |
| Map<PathFragment, Artifact> pathToLegalOutputArtifact, |
| ActionExecutionMetadata actionExecutionMetadata, |
| ActionExecutionContext actionExecutionContext, |
| Set<ArtifactWithInclusionContext> visitedInclusions, |
| Artifact grepIncludes) |
| throws IOException, ExecException, InterruptedException { |
| // Try to find the included file relative to the file that contains the inclusion. Relative |
| // inclusions are handled like the first entry on the quote include path |
| Artifact includeFile = |
| locateRelative(inclusion.getInclusion(), pathToLegalOutputArtifact, source); |
| int contextPathPos = 0; |
| Kind contextKind = null; |
| |
| checkForInterrupt("visiting", source); |
| |
| // If nothing has been found, get an inclusion from the cache. This will automatically search |
| // on the include paths and populate the cache if necessary. |
| if (includeFile == null) { |
| LocateOnPathResult result = inclusionCache.lookup(inclusion, pathToLegalOutputArtifact); |
| includeFile = result.path; |
| contextPathPos = result.includePosition; |
| contextKind = inclusion.getContextKind(); |
| } |
| |
| // Recursively process the found file (if not yet done). |
| if (includeFile != null |
| && !isIllegalOutputFile(includeFile.getExecPath(), pathToLegalOutputArtifact.keySet()) |
| && visitedInclusions.add( |
| new ArtifactWithInclusionContext(includeFile, contextKind, contextPathPos))) { |
| visited.add(includeFile); |
| if (modularHeaders.contains(includeFile)) { |
| return; |
| } |
| processAsyncIfNotExtracted( |
| includeFile, |
| contextPathPos, |
| contextKind, |
| visited, |
| pathToLegalOutputArtifact, |
| actionExecutionMetadata, |
| actionExecutionContext, |
| visitedInclusions, |
| grepIncludes); |
| } |
| } |
| |
| /** |
| * Processes a given list of includes for a given base file and populates the provided set with |
| * the visited includes |
| * |
| * @param source the source file used as a reference for finding includes |
| * @param includes the list of -include option strings to locate and process |
| * @param visited the set of files that are transitively included by {@code includes} to |
| * populate |
| * @param pathToLegalOutputArtifact map to look up legal output artifact by path |
| * @param actionExecutionContext Services in the scope of the action, like the stream to which |
| * @param visitedInclusions the set of all processed inclusions, to avoid processing duplicate |
| * inclusions. |
| */ |
| private void processCmdlineIncludes( |
| Artifact source, |
| List<String> includes, |
| Artifact grepIncludes, |
| Set<Artifact> visited, |
| Map<PathFragment, Artifact> pathToLegalOutputArtifact, |
| ActionExecutionMetadata actionExecutionMetadata, |
| ActionExecutionContext actionExecutionContext, |
| Set<ArtifactWithInclusionContext> visitedInclusions) |
| throws IOException, ExecException, InterruptedException { |
| for (String incl : includes) { |
| InclusionWithContext inclusion = new InclusionWithContext(incl, Kind.QUOTE); |
| findAndProcess( |
| inclusion, |
| source, |
| visited, |
| pathToLegalOutputArtifact, |
| actionExecutionMetadata, |
| actionExecutionContext, |
| visitedInclusions, |
| grepIncludes); |
| } |
| } |
| |
| /** |
| * Processes a bunch sources asynchronously and adds them and their included files to the |
| * provided set. |
| * |
| * @param sources the files to process and add to the set |
| * @param visited the set to receive the files that are transitively included by {@code sources} |
| * @param pathToLegalOutputArtifact map to look up legal output artifact by path |
| * @param actionExecutionContext Services in the scope of the action, like the stream to which |
| * @param visitedInclusions the set of all processed inclusions, to avoid processing duplicate |
| * inclusions. |
| */ |
| private void processBulkAsync( |
| Collection<Artifact> sources, |
| final Set<Artifact> visited, |
| final Map<PathFragment, Artifact> pathToLegalOutputArtifact, |
| final ActionExecutionMetadata actionExecutionMetadata, |
| final ActionExecutionContext actionExecutionContext, |
| final Set<ArtifactWithInclusionContext> visitedInclusions, |
| Artifact grepIncludes) |
| throws IOException, ExecException, InterruptedException { |
| for (final Artifact source : sources) { |
| // TODO(djasper): This looks suspicious. We should only stop based on visitedInclusions. |
| if (!visited.add(source)) { |
| continue; |
| } |
| |
| processAsyncIfNotExtracted( |
| source, |
| /*contextPathPos=*/ -1, |
| /*contextKind=*/ null, |
| visited, |
| pathToLegalOutputArtifact, |
| actionExecutionMetadata, |
| actionExecutionContext, |
| visitedInclusions, |
| grepIncludes); |
| } |
| } |
| |
| private void processFileLevelHintsAsync( |
| final Hints hints, |
| final Artifact include, |
| final Set<Artifact> alsoVisited, |
| final Map<PathFragment, Artifact> pathToLegalOutputArtifact, |
| final ActionExecutionMetadata actionExecutionMetadata, |
| final ActionExecutionContext actionExecutionContext, |
| final Set<ArtifactWithInclusionContext> visitedInclusions, |
| Artifact grepIncludes) { |
| Collection<Artifact> sources = hints.getFileLevelHintedInclusionsLegacy(include); |
| // Early-out if there's nothing to do to avoid enqueuing a closure |
| if (sources.isEmpty()) { |
| return; |
| } |
| super.execute(() -> { |
| try { |
| processBulkAsync( |
| sources, |
| alsoVisited, |
| pathToLegalOutputArtifact, |
| actionExecutionMetadata, |
| actionExecutionContext, |
| visitedInclusions, |
| grepIncludes); |
| } catch (IOException e) { |
| throw new IORuntimeException(e); |
| } catch (ExecException e) { |
| throw new ExecRuntimeException(e); |
| } catch (InterruptedException e) { |
| throw new InterruptedRuntimeException(e); |
| } |
| }); |
| } |
| } |
| |
| private static class ExecRuntimeException extends RuntimeException { |
| private final ExecException cause; |
| |
| public ExecRuntimeException(ExecException e) { |
| super(e); |
| this.cause = e; |
| } |
| |
| public ExecException getRealCause() { |
| return cause; |
| } |
| } |
| |
| private static class InterruptedRuntimeException extends RuntimeException { |
| private final InterruptedException cause; |
| |
| public InterruptedRuntimeException(InterruptedException e) { |
| super(e); |
| this.cause = e; |
| } |
| |
| public InterruptedException getRealCause() { |
| return cause; |
| } |
| } |
| } |
