| // Copyright 2018 The Bazel Authors. All rights reserved. | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| package com.google.devtools.build.lib.includescanning; | |
| import com.google.common.base.Preconditions; | |
| import com.google.common.base.Supplier; | |
| import com.google.common.collect.ImmutableList; | |
| import com.google.common.collect.ImmutableSet; | |
| import com.google.common.collect.Sets; | |
| import com.google.devtools.build.lib.actions.ActionExecutionContext; | |
| import com.google.devtools.build.lib.actions.ActionExecutionMetadata; | |
| import com.google.devtools.build.lib.actions.Artifact; | |
| import com.google.devtools.build.lib.actions.ArtifactFactory; | |
| import com.google.devtools.build.lib.actions.ArtifactRoot; | |
| import com.google.devtools.build.lib.actions.ExecException; | |
| import com.google.devtools.build.lib.actions.MissingDepException; | |
| import com.google.devtools.build.lib.analysis.BlazeDirectories; | |
| import com.google.devtools.build.lib.cmdline.RepositoryName; | |
| import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor; | |
| import com.google.devtools.build.lib.concurrent.ErrorClassifier; | |
| import com.google.devtools.build.lib.concurrent.ThreadSafety; | |
| import com.google.devtools.build.lib.includescanning.IncludeParser.Hints; | |
| import com.google.devtools.build.lib.includescanning.IncludeParser.Inclusion; | |
| import com.google.devtools.build.lib.includescanning.IncludeParser.Inclusion.Kind; | |
| import com.google.devtools.build.lib.rules.cpp.IncludeScanner; | |
| import com.google.devtools.build.lib.vfs.IORuntimeException; | |
| import com.google.devtools.build.lib.vfs.Path; | |
| import com.google.devtools.build.lib.vfs.PathFragment; | |
| import com.google.devtools.build.lib.vfs.Root; | |
| import com.google.devtools.build.skyframe.SkyFunction; | |
| import java.io.IOException; | |
| import java.util.ArrayList; | |
| import java.util.Collection; | |
| import java.util.Collections; | |
| import java.util.Iterator; | |
| import java.util.List; | |
| import java.util.Map; | |
| import java.util.Objects; | |
| import java.util.Random; | |
| import java.util.Set; | |
| import java.util.concurrent.ConcurrentHashMap; | |
| import java.util.concurrent.ConcurrentMap; | |
| import java.util.concurrent.ExecutorService; | |
| /** | |
| * C include scanner. Quickly scans C/C++ source files to determine the bounding set of transitively | |
| * referenced include files. | |
| * | |
| * <p>Maintains caches for parses and search-matches for performance. | |
| * | |
| * <pre> | |
| * TODO(bazel-team): (2009) Currently does not evaluate preprocessor symbols, so computed includes | |
| * are ignored. | |
| * TODO(bazel-team): (2009) Does not handle multiline block comments preceding or around an #include | |
| * </pre> | |
| */ | |
| public class LegacyIncludeScanner implements IncludeScanner { | |
| private static final class ArtifactWithInclusionContext { | |
| private final Artifact artifact; | |
| private final Kind contextKind; | |
| private final int contextPathPos; | |
| private ArtifactWithInclusionContext(Artifact artifact, Kind contextKind, int contextPathPos) { | |
| this.artifact = artifact; | |
| this.contextKind = contextKind; | |
| this.contextPathPos = contextPathPos; | |
| } | |
| @Override | |
| public int hashCode() { | |
| return contextPathPos + 37 * Objects.hash(contextKind, artifact); | |
| } | |
| @Override | |
| public boolean equals(Object obj) { | |
| if (this == obj) { | |
| return true; | |
| } | |
| if (!(obj instanceof ArtifactWithInclusionContext)) { | |
| return false; | |
| } | |
| ArtifactWithInclusionContext that = (ArtifactWithInclusionContext) obj; | |
| return this.contextKind == that.contextKind | |
| && this.contextPathPos == that.contextPathPos | |
| && this.artifact.equals(that.artifact); | |
| } | |
| } | |
| /** | |
| * A cache of inclusion lookups, taking care to avoid spurious caching related | |
| * to generated headers / source files. | |
| */ | |
| @ThreadSafety.ThreadSafe | |
| private class InclusionCache { | |
| private final ConcurrentMap<InclusionWithContext, LocateOnPathResult> cache = | |
| new ConcurrentHashMap<>(); | |
| /** | |
| * Locates an included file along the search paths. The result is cacheable. | |
| * | |
| * @param inclusion the inclusion to locate | |
| * @param pathToLegalOutputArtifact generated files which may be reached during scanning | |
| * @param onlyCheckGenerated if true, only search for generated output files | |
| * @return a tuple of the found file, the position of the respective include path entry on the | |
| * search path (or null if no matching file was found), and whether the scan touched illegal | |
| * output files | |
| */ | |
| private LocateOnPathResult locateOnPaths( | |
| InclusionWithContext inclusion, | |
| Map<PathFragment, Artifact> pathToLegalOutputArtifact, | |
| boolean onlyCheckGenerated) { | |
| PathFragment name = inclusion.getInclusion().pathFragment; | |
| // For #include_next directives we start searching on the include path where | |
| // we found the previous inclusion. | |
| int searchStart = inclusion.getInclusion().kind.isNext() ? inclusion.getContextPathPos() : 0; | |
| // Search the header on the remaining paths. | |
| List<PathFragment> paths = | |
| inclusion.getContextKind() == Kind.QUOTE ? quoteIncludePaths : includePaths; | |
| boolean viewedIllegalOutput = false; | |
| for (int i = searchStart; i < paths.size(); ++i) { | |
| PathFragment fileFragment = paths.get(i).getRelative(name); | |
| if (fileFragment.containsUplevelReferences()) { | |
| // TODO(janakr): This branch shouldn't be necessary: we should be able to filter such | |
| // inclusions out unconditionally. | |
| // Deal with fragments that escape the execroot. They most likely come right back in. | |
| Path execRootRelativePath = execRoot.getRelative(fileFragment); | |
| if (execRootRelativePath.startsWith(execRoot)) { | |
| // Common case: transform #include "../execroot/foo.h" into #include "foo.h" | |
| fileFragment = execRootRelativePath.relativeTo(execRoot); | |
| } else { | |
| // Ugh: successfully escaped the exec root. It's their funeral. | |
| fileFragment = execRootRelativePath.asFragment(); | |
| } | |
| // This can happen when we are processing Windows paths with backslashes on Unix, | |
| // since we do not do any #ifdef processing. | |
| // We can safely discard these here. | |
| if (fileFragment.containsUplevelReferences()) { | |
| continue; | |
| } | |
| } | |
| if (onlyCheckGenerated && !isRealOutputFile(fileFragment)) { | |
| continue; | |
| } | |
| viewedIllegalOutput = | |
| viewedIllegalOutput | |
| || isIllegalOutputFile(fileFragment, pathToLegalOutputArtifact.keySet()); | |
| boolean isOutputDirectory = fileFragment.startsWith(outputPathFragment); | |
| if (!isFile(fileFragment, name, !isOutputDirectory, pathToLegalOutputArtifact.keySet())) { | |
| continue; | |
| } | |
| Artifact artifact; | |
| if (isOutputDirectory) { | |
| // May be a normal output file or an inc_library header. | |
| artifact = pathToLegalOutputArtifact.get(fileFragment); | |
| if (artifact == null) { | |
| // This happens if an included file exists in a cc_inc_library's output directory, | |
| // but is not an output of the cc_inc_library. This can happen if, for instance, the | |
| // definition of the cc_inc_library is changed to output different files, but the | |
| // source file's includes don't change. | |
| // Often, such an include is conditional, and so failing to find it here will not | |
| // lead to problems. If this include is actually needed for compilation, then we will | |
| // emit a somewhat unhelpful error message of a missing file, rather than the more | |
| // helpful one of an illegal include, but it's hard to emit the illegal include | |
| // message consistently, and this is a rare occurrence in any case. | |
| return LocateOnPathResult.createNotFound(viewedIllegalOutput); | |
| } | |
| } else if (!fileFragment.isAbsolute()) { | |
| artifact = artifactFactory.resolveSourceArtifact(fileFragment, RepositoryName.MAIN); | |
| if (artifact == null) { | |
| // There was a real file, but we couldn't resolve it, probably because it belonged to | |
| // a package that wasn't actually loaded this build, so user cannot refer to files in | |
| // that package. | |
| continue; | |
| } | |
| } else { | |
| // This file is given with an absolute path. We will error out after transitive scanning | |
| // of the top-level source is finished unless this corresponds to a built-in include | |
| // directory, and will ignore this artifact in any case, but track it here so that its | |
| // includes can be processed. | |
| artifact = artifactFactory.getSourceArtifact(fileFragment, absoluteRoot); | |
| } | |
| // +1 to account for the virtual entry for relative includes. | |
| return LocateOnPathResult.create(artifact, i + 1, viewedIllegalOutput); | |
| } | |
| // Not found. | |
| return LocateOnPathResult.createNotFound(viewedIllegalOutput); | |
| } | |
| /** | |
| * Locates an included file along the search paths. | |
| * | |
| * @param inclusion the inclusion to locate | |
| * @param pathToLegalOutputArtifact generated files which may be reached during scanning | |
| * @return a LocateOnPathResult | |
| */ | |
| public LocateOnPathResult lookup( | |
| InclusionWithContext inclusion, Map<PathFragment, Artifact> pathToLegalOutputArtifact) { | |
| LocateOnPathResult result = | |
| cache.computeIfAbsent( | |
| inclusion, key -> locateOnPaths(key, pathToLegalOutputArtifact, false)); | |
| // It is not safe to cache lookups which viewed illegal output files. Their nonexistence do | |
| // not imply nonexistence for actions using the same include scanner, but executed later on. | |
| // See bug 2097998. For performance reasons, take a small shortcut: only avoid caching when | |
| // the path lookup result from locateOnPaths() is empty. | |
| if (result.path != null || !result.viewedIllegalOutputFile) { | |
| return result; | |
| } | |
| result = locateOnPaths(inclusion, pathToLegalOutputArtifact, true); | |
| if (result.path != null || !result.viewedIllegalOutputFile) { | |
| // In this case, the result is now cachable either because a file has been found or | |
| // because there are no more illegal output files. This is rare in practice. Avoid | |
| // creating a future and modifying the cache in the common case. | |
| cache.put(inclusion, result); | |
| } | |
| return result; | |
| } | |
| } | |
| private static class LocateOnPathResult { | |
| private static final LocateOnPathResult NOT_FOUND_VIEWED_ILLEGAL = | |
| new LocateOnPathResult(null, -1, true); | |
| private static final LocateOnPathResult NOT_FOUND_NO_VIEWED_ILLEGAL = | |
| new LocateOnPathResult(null, -1, false); | |
| private final Artifact path; | |
| private final int includePosition; | |
| private final boolean viewedIllegalOutputFile; | |
| private LocateOnPathResult( | |
| Artifact path, int includePosition, boolean viewedIllegalOutputFile) { | |
| this.path = path; | |
| this.includePosition = includePosition; | |
| this.viewedIllegalOutputFile = viewedIllegalOutputFile; | |
| } | |
| static LocateOnPathResult create( | |
| Artifact path, int includePosition, boolean viewedIllegalOutputFile) { | |
| return new LocateOnPathResult( | |
| Preconditions.checkNotNull(path), includePosition, viewedIllegalOutputFile); | |
| } | |
| static LocateOnPathResult createNotFound(boolean viewedIllegalOutputFile) { | |
| return viewedIllegalOutputFile ? NOT_FOUND_VIEWED_ILLEGAL : NOT_FOUND_NO_VIEWED_ILLEGAL; | |
| } | |
| } | |
| private final Path execRoot; | |
| private final ArtifactFactory artifactFactory; | |
| private final Supplier<SpawnIncludeScanner> spawnIncludeScannerSupplier; | |
| /** | |
| * Externally-scoped cache of file path => parsed inclusion set mappings. Saves us from having to | |
| * parse files more than once, and can be shared by scanners with different search paths. | |
| */ | |
| private final ConcurrentMap<Artifact, Collection<Inclusion>> fileParseCache; | |
| private final IncludeParser parser; | |
| /** | |
| * Search path for searching for all quoted "xyz.h" includes, composed of all the -iquote, -I and | |
| * -isystem paths (in this order). | |
| */ | |
| private final ImmutableList<PathFragment> quoteIncludePaths; | |
| /** | |
| * Search path for searching for all includes, composed of all the -I and -isystem paths (in this | |
| * order). | |
| */ | |
| private final List<PathFragment> includePaths; | |
| private final PathFragment includeRootFragment; | |
| private final PathFragment outputPathFragment; | |
| private final Root absoluteRoot; | |
| /** | |
| * Scanner-scoped cache of inclusions with their resolved files and include path entries. This | |
| * cache is specific to a given pair of search paths, and is thus scanner-local. | |
| * | |
| * <p>Each inclusion (name+type+context) is associated with its resolved file here, thus saving | |
| * redundant path searches. The second entry of the pair is the include path entry on which the | |
| * file was found. | |
| */ | |
| private final InclusionCache inclusionCache; | |
| private final PathExistenceCache pathCache; | |
| private final ExecutorService includePool; | |
| // We are using this Random just for shuffling, so keep the order deterministic by hardcoding | |
| // the seed. | |
| private static final Random CONSTANT_SEED_RANDOM = new Random(88); | |
| /** | |
| * Constructs a new IncludeScanner | |
| * | |
| * @param cache externally scoped cache of file-path to inclusion-set mappings | |
| * @param pathCache include path existence cache | |
| * @param quoteIncludePaths the list of quote search path dirs (-iquote) | |
| * @param includePaths the list of all other search path dirs (-I and -isystem) | |
| */ | |
| LegacyIncludeScanner( | |
| IncludeParser parser, | |
| ExecutorService includePool, | |
| ConcurrentMap<Artifact, Collection<Inclusion>> cache, | |
| PathExistenceCache pathCache, | |
| List<PathFragment> quoteIncludePaths, | |
| List<PathFragment> includePaths, | |
| Path outputPath, | |
| Path execRoot, | |
| ArtifactFactory artifactFactory, | |
| Supplier<SpawnIncludeScanner> spawnIncludeScannerSupplier) { | |
| this.parser = parser; | |
| this.includePool = includePool; | |
| this.fileParseCache = cache; | |
| this.pathCache = pathCache; | |
| this.artifactFactory = Preconditions.checkNotNull(artifactFactory); | |
| this.spawnIncludeScannerSupplier = spawnIncludeScannerSupplier; | |
| this.quoteIncludePaths = ImmutableList.<PathFragment>builder() | |
| .addAll(quoteIncludePaths) | |
| .addAll(includePaths) | |
| .build(); | |
| this.includePaths = ImmutableList.copyOf(includePaths); | |
| this.inclusionCache = new InclusionCache(); | |
| this.execRoot = execRoot; | |
| this.outputPathFragment = outputPath.relativeTo(execRoot); | |
| this.includeRootFragment = | |
| outputPathFragment.getRelative(BlazeDirectories.RELATIVE_INCLUDE_DIR); | |
| this.absoluteRoot = Root.absoluteRoot(execRoot.getFileSystem()); | |
| } | |
| /** | |
| * Locates an included file relative to the including file. The result is not cacheable. | |
| * | |
| * @param inclusion the inclusion to locate | |
| * @param includer the including file | |
| * @return the resolved Path, or null if no file could be found | |
| */ | |
| private Artifact locateRelative( | |
| Inclusion inclusion, Map<PathFragment, Artifact> legalOutputFiles, Artifact includer) { | |
| if (inclusion.kind != Kind.QUOTE) { | |
| return null; | |
| } | |
| PathFragment name = inclusion.pathFragment; | |
| PathFragment execPath = includer.getExecPath().getParentDirectory().getRelative(name); | |
| if (!isFile(execPath, name, includer.isSourceArtifact(), legalOutputFiles.keySet())) { | |
| return null; | |
| } | |
| PathFragment parentDirectory = includer.getRootRelativePath().getParentDirectory(); | |
| PathFragment rootRelativePath = parentDirectory.getRelative(name); | |
| if (rootRelativePath.containsUplevelReferences()) { | |
| // An include cannot break out of a (package path) root via a relative inclusion. It should | |
| // also not break out of the root and then come back into it -- who knows what hardcoded | |
| // directory names there are in it. | |
| return null; | |
| } | |
| if (legalOutputFiles.containsKey(execPath)) { | |
| return legalOutputFiles.get(execPath); | |
| } | |
| ArtifactRoot root = includer.getRoot(); | |
| Artifact sourceArtifact = | |
| artifactFactory.resolveSourceArtifactWithAncestor( | |
| name, parentDirectory, root, RepositoryName.MAIN); | |
| if (sourceArtifact == null) { | |
| // If the name had up-level references, this path may not be under any package. Otherwise, | |
| // we must have gotten an artifact, since it should be under the same package as the | |
| // including artifact. | |
| Preconditions.checkState( | |
| name.containsUplevelReferences(), | |
| "%s %s %s %s", | |
| name, | |
| parentDirectory, | |
| rootRelativePath, | |
| root); | |
| } | |
| return sourceArtifact; | |
| } | |
| /** Returns whether the given path exists in the filesystem. */ | |
| private boolean isFile( | |
| PathFragment execPath, PathFragment includeAsWritten, boolean isSource, | |
| Collection<PathFragment> legalOutputFiles) { | |
| if (isRealOutputFile(execPath)) { | |
| return legalOutputFiles.contains(execPath); | |
| } | |
| // TODO(djasper): This code path cannot be hit with isSource being false. Verify and add | |
| // Preconditions check. | |
| if (isSource && !execPath.isAbsolute() && execPath.endsWith(includeAsWritten)) { | |
| // Verify that the directory of execPath exists as an optimization. Most includes are relative | |
| // to the workspace and we'd like to avoid stat'ing every such include relative to every | |
| // include path. If testing whether "a/b/c.h" is a file beneath the include path "e/f/", | |
| // verify that "e/f/a" and "e/f/a/b" are valid directories (and cache the result). | |
| int execPathSegments = execPath.segmentCount(); | |
| int nameSegments = includeAsWritten.segmentCount(); | |
| for (int i = execPathSegments - nameSegments + 1; i < execPathSegments; i++) { | |
| if (!pathCache.directoryExists(execPath.subFragment(0, i))) { | |
| return false; | |
| } | |
| } | |
| } | |
| return pathCache.fileExists(execPath, isSource); | |
| } | |
| @Override | |
| public void process( | |
| Artifact mainSource, | |
| Collection<Artifact> sources, | |
| IncludeScanningHeaderData includeScanningHeaderData, | |
| List<String> cmdlineIncludes, | |
| Set<Artifact> includes, | |
| ActionExecutionMetadata actionExecutionMetadata, | |
| ActionExecutionContext actionExecutionContext, | |
| Artifact grepIncludes) | |
| throws IOException, ExecException, InterruptedException { | |
| try { | |
| // Because our IncludeVisitor can dynamically switch to in-thread execution, we could get | |
| // unlucky and run one of the async calls on the main thread. Catch and rethrow the runtime | |
| // exceptions as the appropriate corresponding checked exception. | |
| processInternal( | |
| mainSource, | |
| sources, | |
| includeScanningHeaderData, | |
| cmdlineIncludes, | |
| includes, | |
| actionExecutionMetadata, | |
| actionExecutionContext, | |
| grepIncludes); | |
| } catch (IORuntimeException e) { | |
| throw e.getCauseIOException(); | |
| } catch (ExecRuntimeException e) { | |
| throw e.getRealCause(); | |
| } catch (InterruptedRuntimeException e) { | |
| throw e.getRealCause(); | |
| } | |
| } | |
| private void processInternal( | |
| Artifact mainSource, | |
| Collection<Artifact> sources, | |
| IncludeScanningHeaderData includeScanningHeaderData, | |
| List<String> cmdlineIncludes, | |
| Set<Artifact> includes, | |
| ActionExecutionMetadata actionExecutionMetadata, | |
| ActionExecutionContext actionExecutionContext, | |
| Artifact grepIncludes) | |
| throws IOException, ExecException, InterruptedException { | |
| Preconditions.checkArgument(mainSource == null || sources.contains(mainSource), | |
| "The main source '%s' is not part of '%s'", mainSource, sources); | |
| ImmutableSet.Builder<Artifact> pathHints = null; | |
| SkyFunction.Environment env = actionExecutionContext.getEnvironmentForDiscoveringInputs(); | |
| if (parser.getHints() != null) { | |
| pathHints = ImmutableSet.builderWithExpectedSize(quoteIncludePaths.size()); | |
| Collection<Artifact> artifacts = | |
| parser.getHints().getPathLevelHintedInclusions(quoteIncludePaths, env); | |
| if (!env.valuesMissing()) { | |
| pathHints.addAll(Preconditions.checkNotNull(artifacts, quoteIncludePaths)); | |
| } | |
| } | |
| if (env.valuesMissing()) { | |
| throw new MissingDepException(); | |
| } | |
| Set<ArtifactWithInclusionContext> visitedInclusions = Sets.newConcurrentHashSet(); | |
| IncludeVisitor visitor = new IncludeVisitor(includeScanningHeaderData.getModularHeaders()); | |
| try { | |
| // Process cmd line includes, if specified. | |
| if (mainSource != null && !cmdlineIncludes.isEmpty()) { | |
| visitor.processCmdlineIncludes( | |
| mainSource, | |
| cmdlineIncludes, | |
| grepIncludes, | |
| includes, | |
| includeScanningHeaderData.getPathToLegalOutputArtifact(), | |
| actionExecutionMetadata, | |
| actionExecutionContext, | |
| visitedInclusions); | |
| visitor.sync(); | |
| } | |
| visitor.processBulkAsync( | |
| sources, | |
| includes, | |
| includeScanningHeaderData.getPathToLegalOutputArtifact(), | |
| actionExecutionMetadata, | |
| actionExecutionContext, | |
| visitedInclusions, | |
| grepIncludes); | |
| visitor.sync(); | |
| // Process include hints | |
| // TODO(ulfjack): Make this code go away. Use the new hinted inclusions instead. | |
| Hints hints = parser.getHints(); | |
| if (hints != null) { | |
| // Follow "path" hints. | |
| visitor.processBulkAsync( | |
| pathHints.build(), | |
| includes, | |
| includeScanningHeaderData.getPathToLegalOutputArtifact(), | |
| actionExecutionMetadata, | |
| actionExecutionContext, | |
| visitedInclusions, | |
| grepIncludes); | |
| // Follow "file" hints for the primary sources. | |
| for (Artifact source : sources) { | |
| visitor.processFileLevelHintsAsync( | |
| hints, | |
| source, | |
| includes, | |
| includeScanningHeaderData.getPathToLegalOutputArtifact(), | |
| actionExecutionMetadata, | |
| actionExecutionContext, | |
| visitedInclusions, | |
| grepIncludes); | |
| } | |
| visitor.sync(); | |
| // Follow "file" hints for all included headers, transitively. | |
| Set<Artifact> frontier = includes; | |
| while (!frontier.isEmpty()) { | |
| Set<Artifact> adjacent = Sets.newConcurrentHashSet(); | |
| for (Artifact include : frontier) { | |
| visitor.processFileLevelHintsAsync( | |
| hints, | |
| include, | |
| adjacent, | |
| includeScanningHeaderData.getPathToLegalOutputArtifact(), | |
| actionExecutionMetadata, | |
| actionExecutionContext, | |
| visitedInclusions, | |
| grepIncludes); | |
| } | |
| visitor.sync(); | |
| // Keep novel nodes as the next frontier. | |
| for (Iterator<Artifact> iter = adjacent.iterator(); iter.hasNext(); ) { | |
| if (!includes.add(iter.next())) { | |
| iter.remove(); | |
| } | |
| } | |
| frontier = adjacent; | |
| } | |
| } | |
| } catch (IOException | InterruptedException | ExecException | MissingDepException e) { | |
| // Careful: Do not leak visitation threads if we have an exception in the initial thread. | |
| visitor.sync(); | |
| throw e; | |
| } | |
| } | |
| private static void checkForInterrupt(String operation, Object source) | |
| throws InterruptedException { | |
| // We require passing in the operation and the source Path / Artifact to avoid intermediate | |
| // String operations. The include scanner is performance critical and this showed up in a | |
| // profiler. | |
| if (Thread.currentThread().isInterrupted()) { | |
| throw new InterruptedException( | |
| "Include scanning interrupted while " + operation + " " + source); | |
| } | |
| } | |
| private boolean isIllegalOutputFile( | |
| PathFragment includeFile, Collection<PathFragment> legalOutputFiles) { | |
| return isRealOutputFile(includeFile) && !legalOutputFiles.contains(includeFile); | |
| } | |
| private boolean isRealOutputFile(PathFragment path) { | |
| return path.startsWith(outputPathFragment) && !isIncPath(path); | |
| } | |
| private boolean isIncPath(PathFragment path) { | |
| // See CreateIncSymlinkAction and where it's used: The symlink trees | |
| // are always rooted at locations that fit the logic here. | |
| return path.startsWith(includeRootFragment) && !path.equals(includeRootFragment); | |
| } | |
| /** | |
| * Implements a potentially parallel traversal over source files using a | |
| * thread pool shared across different IncludeScanner instances. | |
| */ | |
| private class IncludeVisitor extends AbstractQueueVisitor { | |
| /** The set of headers known to be part of a C++ module. Scanning can stop here. */ | |
| private Set<Artifact> modularHeaders; | |
| public IncludeVisitor(Set<Artifact> modularHeaders) { | |
| super( | |
| includePool, | |
| /*shutdownOnCompletion=*/ false, | |
| /*failFastOnException=*/ true, | |
| ErrorClassifier.DEFAULT); | |
| this.modularHeaders = modularHeaders; | |
| } | |
| /** | |
| * Block for the completion of all outstanding visitations. | |
| */ | |
| public void sync() throws IOException, ExecException, InterruptedException { | |
| try { | |
| super.awaitQuiescence(true); | |
| } catch (InterruptedException e) { | |
| throw new InterruptedException("Interrupted during include visitation"); | |
| } catch (IORuntimeException e) { | |
| throw e.getCauseIOException(); | |
| } catch (ExecRuntimeException e) { | |
| throw e.getRealCause(); | |
| } catch (InterruptedRuntimeException e) { | |
| throw e.getRealCause(); | |
| } | |
| } | |
| /** | |
| * Processes a given file for includes and populates the provided set with the visited includes. | |
| * | |
| * @param source the file to process | |
| * @param contextPathPos the position on the include path where the containing file was found, | |
| * or <code>-1</code> for top-level inclusions | |
| * @param contextKind the kind how the containing file was included, or null for top-level | |
| * inclusions | |
| * @param visited the set to receive the files that are transitively included by {@code source} | |
| * @param pathToLegalOutputArtifact map to look up legal output artifact by path | |
| * @param actionExecutionMetadata owning action | |
| * @param actionExecutionContext Services in the scope of the action, like the stream to which | |
| * @param visitedInclusions the set of all processed inclusions, to avoid processing duplicate | |
| * inclusions. | |
| */ | |
| private void process( | |
| final Artifact source, | |
| int contextPathPos, | |
| Kind contextKind, | |
| Set<Artifact> visited, | |
| Map<PathFragment, Artifact> pathToLegalOutputArtifact, | |
| final ActionExecutionMetadata actionExecutionMetadata, | |
| final ActionExecutionContext actionExecutionContext, | |
| Set<ArtifactWithInclusionContext> visitedInclusions, | |
| final Artifact grepIncludes) | |
| throws IOException, ExecException, InterruptedException { | |
| checkForInterrupt("processing", source); | |
| Collection<Inclusion> inclusions = null; | |
| try { | |
| inclusions = | |
| fileParseCache.computeIfAbsent( | |
| source, | |
| file -> { | |
| try { | |
| return parser.extractInclusions( | |
| file, | |
| actionExecutionMetadata, | |
| actionExecutionContext, | |
| grepIncludes, | |
| spawnIncludeScannerSupplier.get(), | |
| isRealOutputFile(source.getExecPath())); | |
| } catch (IOException e) { | |
| throw new IORuntimeException(e); | |
| } catch (ExecException e) { | |
| throw new ExecRuntimeException(e); | |
| } catch (InterruptedException e) { | |
| throw new InterruptedRuntimeException(e); | |
| } | |
| }); | |
| } catch (IORuntimeException e) { | |
| throw e.getCauseIOException(); | |
| } catch (ExecRuntimeException e) { | |
| throw e.getRealCause(); | |
| } catch (InterruptedRuntimeException e) { | |
| throw e.getRealCause(); | |
| } | |
| Preconditions.checkNotNull(inclusions, source); | |
| // Shuffle the inclusions to get better parallelism. See b/62200470. | |
| List<Inclusion> shuffledInclusions = new ArrayList<>(inclusions); | |
| Collections.shuffle(shuffledInclusions, CONSTANT_SEED_RANDOM); | |
| // For each inclusion: get or locate its target file & recursively process | |
| IncludeScannerHelper helper = | |
| new IncludeScannerHelper(includePaths, quoteIncludePaths, source); | |
| for (Inclusion inclusion : shuffledInclusions) { | |
| findAndProcess( | |
| helper.createInclusionWithContext(inclusion, contextPathPos, contextKind), | |
| source, | |
| visited, | |
| pathToLegalOutputArtifact, | |
| actionExecutionMetadata, | |
| actionExecutionContext, | |
| visitedInclusions, | |
| grepIncludes); | |
| } | |
| } | |
| /** | |
| * Same as {@link #process}, but executes asynchronously if the #include lines of {@code source} | |
| * haven't been extracted yet. For sources that have already been extracted, just continue | |
| * walking them in the current thread. The overhead of scheduling this on other threads is | |
| * larger than the gain in concurrency. The only really slow operation is the (possibly remote) | |
| * extraction of includes. | |
| */ | |
| private void processAsyncIfNotExtracted( | |
| final Artifact source, | |
| int contextPathPos, | |
| Kind contextKind, | |
| Set<Artifact> visited, | |
| Map<PathFragment, Artifact> pathToLegalOutputArtifact, | |
| final ActionExecutionMetadata actionExecutionMetadata, | |
| final ActionExecutionContext actionExecutionContext, | |
| Set<ArtifactWithInclusionContext> visitedInclusions, | |
| final Artifact grepIncludes) | |
| throws IOException, ExecException, InterruptedException { | |
| Collection<Inclusion> cacheResult = fileParseCache.get(source); | |
| if (cacheResult != null) { | |
| process( | |
| source, | |
| contextPathPos, | |
| contextKind, | |
| visited, | |
| pathToLegalOutputArtifact, | |
| actionExecutionMetadata, | |
| actionExecutionContext, | |
| visitedInclusions, | |
| grepIncludes); | |
| } else { | |
| super.execute(() -> { | |
| try { | |
| process( | |
| source, | |
| contextPathPos, | |
| contextKind, | |
| visited, | |
| pathToLegalOutputArtifact, | |
| actionExecutionMetadata, | |
| actionExecutionContext, | |
| visitedInclusions, | |
| grepIncludes); | |
| } catch (IOException e) { | |
| throw new IORuntimeException(e); | |
| } catch (ExecException e) { | |
| throw new ExecRuntimeException(e); | |
| } catch (InterruptedException e) { | |
| throw new InterruptedRuntimeException(e); | |
| } | |
| }); | |
| } | |
| } | |
| /** Visits an inclusion starting from a source file. */ | |
| private void findAndProcess( | |
| InclusionWithContext inclusion, | |
| Artifact source, | |
| Set<Artifact> visited, | |
| Map<PathFragment, Artifact> pathToLegalOutputArtifact, | |
| ActionExecutionMetadata actionExecutionMetadata, | |
| ActionExecutionContext actionExecutionContext, | |
| Set<ArtifactWithInclusionContext> visitedInclusions, | |
| Artifact grepIncludes) | |
| throws IOException, ExecException, InterruptedException { | |
| // Try to find the included file relative to the file that contains the inclusion. Relative | |
| // inclusions are handled like the first entry on the quote include path | |
| Artifact includeFile = | |
| locateRelative(inclusion.getInclusion(), pathToLegalOutputArtifact, source); | |
| int contextPathPos = 0; | |
| Kind contextKind = null; | |
| checkForInterrupt("visiting", source); | |
| // If nothing has been found, get an inclusion from the cache. This will automatically search | |
| // on the include paths and populate the cache if necessary. | |
| if (includeFile == null) { | |
| LocateOnPathResult result = inclusionCache.lookup(inclusion, pathToLegalOutputArtifact); | |
| includeFile = result.path; | |
| contextPathPos = result.includePosition; | |
| contextKind = inclusion.getContextKind(); | |
| } | |
| // Recursively process the found file (if not yet done). | |
| if (includeFile != null | |
| && !isIllegalOutputFile(includeFile.getExecPath(), pathToLegalOutputArtifact.keySet()) | |
| && visitedInclusions.add( | |
| new ArtifactWithInclusionContext(includeFile, contextKind, contextPathPos))) { | |
| visited.add(includeFile); | |
| if (modularHeaders.contains(includeFile)) { | |
| return; | |
| } | |
| processAsyncIfNotExtracted( | |
| includeFile, | |
| contextPathPos, | |
| contextKind, | |
| visited, | |
| pathToLegalOutputArtifact, | |
| actionExecutionMetadata, | |
| actionExecutionContext, | |
| visitedInclusions, | |
| grepIncludes); | |
| } | |
| } | |
| /** | |
| * Processes a given list of includes for a given base file and populates the provided set with | |
| * the visited includes | |
| * | |
| * @param source the source file used as a reference for finding includes | |
| * @param includes the list of -include option strings to locate and process | |
| * @param visited the set of files that are transitively included by {@code includes} to | |
| * populate | |
| * @param pathToLegalOutputArtifact map to look up legal output artifact by path | |
| * @param actionExecutionContext Services in the scope of the action, like the stream to which | |
| * @param visitedInclusions the set of all processed inclusions, to avoid processing duplicate | |
| * inclusions. | |
| */ | |
| private void processCmdlineIncludes( | |
| Artifact source, | |
| List<String> includes, | |
| Artifact grepIncludes, | |
| Set<Artifact> visited, | |
| Map<PathFragment, Artifact> pathToLegalOutputArtifact, | |
| ActionExecutionMetadata actionExecutionMetadata, | |
| ActionExecutionContext actionExecutionContext, | |
| Set<ArtifactWithInclusionContext> visitedInclusions) | |
| throws IOException, ExecException, InterruptedException { | |
| for (String incl : includes) { | |
| InclusionWithContext inclusion = new InclusionWithContext(incl, Kind.QUOTE); | |
| findAndProcess( | |
| inclusion, | |
| source, | |
| visited, | |
| pathToLegalOutputArtifact, | |
| actionExecutionMetadata, | |
| actionExecutionContext, | |
| visitedInclusions, | |
| grepIncludes); | |
| } | |
| } | |
| /** | |
| * Processes a bunch sources asynchronously and adds them and their included files to the | |
| * provided set. | |
| * | |
| * @param sources the files to process and add to the set | |
| * @param visited the set to receive the files that are transitively included by {@code sources} | |
| * @param pathToLegalOutputArtifact map to look up legal output artifact by path | |
| * @param actionExecutionContext Services in the scope of the action, like the stream to which | |
| * @param visitedInclusions the set of all processed inclusions, to avoid processing duplicate | |
| * inclusions. | |
| */ | |
| private void processBulkAsync( | |
| Collection<Artifact> sources, | |
| final Set<Artifact> visited, | |
| final Map<PathFragment, Artifact> pathToLegalOutputArtifact, | |
| final ActionExecutionMetadata actionExecutionMetadata, | |
| final ActionExecutionContext actionExecutionContext, | |
| final Set<ArtifactWithInclusionContext> visitedInclusions, | |
| Artifact grepIncludes) | |
| throws IOException, ExecException, InterruptedException { | |
| for (final Artifact source : sources) { | |
| // TODO(djasper): This looks suspicious. We should only stop based on visitedInclusions. | |
| if (!visited.add(source)) { | |
| continue; | |
| } | |
| processAsyncIfNotExtracted( | |
| source, | |
| /*contextPathPos=*/ -1, | |
| /*contextKind=*/ null, | |
| visited, | |
| pathToLegalOutputArtifact, | |
| actionExecutionMetadata, | |
| actionExecutionContext, | |
| visitedInclusions, | |
| grepIncludes); | |
| } | |
| } | |
| private void processFileLevelHintsAsync( | |
| final Hints hints, | |
| final Artifact include, | |
| final Set<Artifact> alsoVisited, | |
| final Map<PathFragment, Artifact> pathToLegalOutputArtifact, | |
| final ActionExecutionMetadata actionExecutionMetadata, | |
| final ActionExecutionContext actionExecutionContext, | |
| final Set<ArtifactWithInclusionContext> visitedInclusions, | |
| Artifact grepIncludes) { | |
| Collection<Artifact> sources = hints.getFileLevelHintedInclusionsLegacy(include); | |
| // Early-out if there's nothing to do to avoid enqueuing a closure | |
| if (sources.isEmpty()) { | |
| return; | |
| } | |
| super.execute(() -> { | |
| try { | |
| processBulkAsync( | |
| sources, | |
| alsoVisited, | |
| pathToLegalOutputArtifact, | |
| actionExecutionMetadata, | |
| actionExecutionContext, | |
| visitedInclusions, | |
| grepIncludes); | |
| } catch (IOException e) { | |
| throw new IORuntimeException(e); | |
| } catch (ExecException e) { | |
| throw new ExecRuntimeException(e); | |
| } catch (InterruptedException e) { | |
| throw new InterruptedRuntimeException(e); | |
| } | |
| }); | |
| } | |
| } | |
| private static class ExecRuntimeException extends RuntimeException { | |
| private final ExecException cause; | |
| public ExecRuntimeException(ExecException e) { | |
| super(e); | |
| this.cause = e; | |
| } | |
| public ExecException getRealCause() { | |
| return cause; | |
| } | |
| } | |
| private static class InterruptedRuntimeException extends RuntimeException { | |
| private final InterruptedException cause; | |
| public InterruptedRuntimeException(InterruptedException e) { | |
| super(e); | |
| this.cause = e; | |
| } | |
| public InterruptedException getRealCause() { | |
| return cause; | |
| } | |
| } | |
| } |