blob: 5c40f9ede39da21fd865d6ad75735b66984dec18 [file] [log] [blame]
// Copyright 2024 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.analysis;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Multimaps;
import com.google.common.collect.Sets;
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.runtime.ConfigFlagDefinitions;
import com.google.devtools.build.lib.server.FailureDetails.BuildConfiguration.Code;
import com.google.devtools.build.lib.skyframe.ProjectFilesLookupValue;
import com.google.devtools.build.lib.skyframe.ProjectValue;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
import com.google.devtools.build.lib.skyframe.config.FlagSetValue;
import com.google.devtools.build.skyframe.EvaluationResult;
import com.google.devtools.build.skyframe.SkyValue;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import javax.annotation.Nullable;
/**
* Container for reading project data.
*
* <p>A "project" is a set of related packages that support a common piece of software. For example,
* "bazel" is a project that includes packages {@code src/main/java/com/google/devtools/build/lib},
* {@code src/main/java/com/google/devtools/build/lib/analysis}, {@code src/test/cpp}, and more.
*
* <p>"Project data" is any useful information that might be associated with a project. Possible
* consumers include <a
* href="https://github.com/bazelbuild/bazel/commit/693215317a6732085731809266f63ff0e7fc31a5">
* Skyfocus</a>> and project-sanctioned build flags (i.e. "these are the correct flags to use with
* this project").
*
* <p>Projects are defined in .scl files that are checked into source control with BUILD files and
* code. scl stands for "Starlark configuration language". This is a limited subset of Starlark
* intended to model generic configuration without requiring Bazel to parse it (similar to JSON).
*
* <p>This is not the same as {@link com.google.devtools.build.lib.runtime.ProjectFile}. That's an
* older implementation of the same idea that was built before .scl and .bzl existed. The code here
* is a rejuvenation of these ideas with more modern APIs.
*/
// TODO: b/324127050 - Make the co-existence of this and ProjectFile less confusing. ProjectFile is
// an outdated API that should be removed.
public final class Project {
private Project() {}
/**
* The active PROJECT.scls for this build.
*
* <p>For example, {@code $ bazel build //foo //bar //baz} could resolve to one, two, or three
* PROJECT.scls, or a mixed state where only some targets have PROJECT.scls.
*
* <p>Consuming code needs to determine if mixed states are valid. People often build multiple
* projects in a single invocation. We don't want to automatically break those builds if there's
* still a sound way to build them.
*
* @param projectFilesToTargetLabels map of PROJECT.scls to the targets that resolve to them.
* @param partialProjectBuild true if some of this build's targets have PROJECT.scls and others
* don't.
* @param differentProjectsDetails A descriptive message explaining how different targets resolve
* to different PROJECT.scls. Consuming code can use this to provide helpful errors if it
* determines the build isn't valid because of this.
*/
public record ActiveProjects(
LinkedHashMap<Label, Collection<Label>> projectFilesToTargetLabels,
boolean partialProjectBuild,
String differentProjectsDetails) {
public boolean isEmpty() {
return projectFilesToTargetLabels.isEmpty();
}
/** User-friendly description of this build type, for consumer info/error messaging. */
public String buildType() {
if (projectFilesToTargetLabels.size() > 1) {
return "multi-project build";
} else if (partialProjectBuild) {
return "build where only some targets have projects";
} else if (projectFilesToTargetLabels.size() == 1) {
return "single-project build";
} else {
return "build with no projects";
}
}
}
/**
* Returns the canonical project files for a set of targets.
*
* <p>If a target matches multiple project files (like {@code a/PROJECT.scl} and {@code
* a/b/PROJECT.scl}), only the innermost is considered.
*
* @param targets targets to resolve project files for
* @param skyframeExecutor support for SkyFunctions that look up project files
* @param eventHandler event handler
* @throws ProjectResolutionException if project resolution fails for any reason
*/
// TODO: b/324127375 - Support hierarchical project files: [foo/project.scl, foo/bar/project.scl].
@Nullable
@VisibleForTesting
public static ActiveProjects getProjectFiles(
Collection<Label> targets,
SkyframeExecutor skyframeExecutor,
ExtendedEventHandler eventHandler)
throws ProjectResolutionException {
// Map targets to their innermost matching project file. Omits targets with no project files.
ImmutableMap<Label, Label> targetsToProjectFiles =
// findProjectFiles returns all project files up a target's path (and omits targets with
// no project files). We just use the first entry, which is the innermost file. For
// example, given [a/b/PROJECT.scl, a/PROJECT.scl], we just use a/b/PROJECT.scl.
findProjectFiles(targets, skyframeExecutor, eventHandler).asMap().entrySet().stream()
.collect(
toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().iterator().next()));
if (targetsToProjectFiles.isEmpty()) {
// None of the targets have project files.
return new ActiveProjects(new LinkedHashMap<>(), /* partialProjectBuild= */ false, "");
}
Set<Label> targetsWithNoProjectFiles =
Sets.difference(ImmutableSet.copyOf(targets), targetsToProjectFiles.keySet());
// Since project files can be aliases to other files, we need to parse them to potentially remap
// to their references. Also remember the targets that resolved to that project file for clean
// error reporting.
LinkedListMultimap<ProjectValue.Key, Label> projectFileKeysToTargets =
Multimaps.invertFrom(
Multimaps.forMap(
targetsToProjectFiles.entrySet().stream()
.collect(
toImmutableMap(
Map.Entry::getKey, entry -> new ProjectValue.Key(entry.getValue())))),
LinkedListMultimap.create());
// Load project file content from Skyframe.
EvaluationResult<SkyValue> evalResult =
skyframeExecutor.evaluateSkyKeys(
eventHandler, projectFileKeysToTargets.keySet(), /* keepGoing= */ false);
if (evalResult.hasError()) {
throw new ProjectResolutionException(
"error loading project files: " + evalResult.getError().getException().getMessage(),
evalResult.getError().getException());
}
// De-duplicate the projectFileKeysToTargets keys by resolving project aliases, and store the
// resulting canonicalized project-to-targets mapping in canonicalProjectToTargets.
LinkedHashMap<Label, Collection<Label>> canonicalProjectsToTargets = new LinkedHashMap<>();
for (var keyToTargets : projectFileKeysToTargets.asMap().entrySet()) {
canonicalProjectsToTargets.put(
((ProjectValue) evalResult.get(keyToTargets.getKey())).getActualProjectFile(),
keyToTargets.getValue());
}
if (canonicalProjectsToTargets.size() != 1) {
// Targets resolve to different project files.
return new ActiveProjects(
canonicalProjectsToTargets,
!canonicalProjectsToTargets.keySet().isEmpty() && !targetsWithNoProjectFiles.isEmpty(),
differentProjectFilesError(canonicalProjectsToTargets, targetsWithNoProjectFiles));
} else {
Label projectFile = Iterables.getOnlyElement(canonicalProjectsToTargets.keySet());
eventHandler.handle(
Event.info(String.format("Reading project settings from %s.", projectFile)));
}
if (targetsWithNoProjectFiles.isEmpty()) {
// All targets resolve to the same canonical project file.
return new ActiveProjects(canonicalProjectsToTargets, false, "");
} else {
// Some targets have project files and some don't.
return new ActiveProjects(canonicalProjectsToTargets, /* partialProjectBuild= */ true, "");
}
}
/**
* User-friendly error message for when targets resolve to different project files or only some
* targets have project files.
*/
private static String differentProjectFilesError(
Map<Label, Collection<Label>> canonicalProjectsToTargets,
Set<Label> targetsWithNoProjectFiles) {
StringBuilder msgBuilder = new StringBuilder("Targets have different project settings:");
// Maximum number of "//foo:target -> //foo:PROJECT.scl" entries to show.
final int maxToShow = 5;
// Iterate through each project file group (and also the "no project file" group), adding one
// entry from each group until we reach the max. This samples each group as evenly as possible.
ListMultimap<Label, Label> groupedResults = LinkedListMultimap.create();
LinkedHashMap<Label, Iterator<Label>> projectFileToNextTarget = new LinkedHashMap<>();
for (var entry : canonicalProjectsToTargets.entrySet()) {
projectFileToNextTarget.put(entry.getKey(), entry.getValue().iterator());
}
if (!targetsWithNoProjectFiles.isEmpty()) {
projectFileToNextTarget.put(null, targetsWithNoProjectFiles.iterator());
}
int totalResults = 0;
LinkedHashMap<Label, Iterator<Label>> nextGroupIteration = projectFileToNextTarget;
while (totalResults < maxToShow && !nextGroupIteration.isEmpty()) {
projectFileToNextTarget = nextGroupIteration;
nextGroupIteration = new LinkedHashMap<>();
for (var entry : projectFileToNextTarget.entrySet()) {
Iterator<Label> nextTarget = entry.getValue();
groupedResults.put(entry.getKey(), nextTarget.next());
if (nextTarget.hasNext()) {
nextGroupIteration.put(entry.getKey(), nextTarget);
}
totalResults++;
if (totalResults == maxToShow) {
break;
}
}
}
// Report results grouped by PROJECT file.
for (var group : groupedResults.asMap().entrySet()) {
String projectFile = group.getKey() == null ? "no project file" : group.getKey().toString();
for (var target : group.getValue()) {
msgBuilder.append(String.format("\n - %s -> %s", target, projectFile));
}
}
int resultsLeft = projectFileToNextTarget.values().stream().mapToInt(Iterators::size).sum();
if (resultsLeft > 0) {
msgBuilder.append(String.format("\n (...and %d more)", resultsLeft));
}
return msgBuilder.toString();
}
/**
* Finds and returns the project files for a set of build targets.
*
* <p>This walks up each target's package path looking for {@link
* com.google.devtools.build.lib.skyframe.ProjectFilesLookupFunction#PROJECT_FILE_NAME} files. For
* example, for {@code //foo/bar/baz:mytarget}, this might look in {@code foo/bar/baz}, {@code
* foo/bar}, and {@code foo} ("might" because it skips directories that don't have BUILD files -
* those directories aren't packages).
*
* <p>This method doesn't read project file content so it doesn't resolve project file aliases.
*
* @return a map from each target to its set of project files, ordered by reverse package depth.
* So a project file in {@code foo/bar} appears before a project file in {@code foo}. Targets
* with no matching project files aren't in the map.
*/
// TODO: b/324127050 - Document resolution semantics when this is less experimental.
@VisibleForTesting
static ImmutableMultimap<Label, Label> findProjectFiles(
Collection<Label> targets,
SkyframeExecutor skyframeExecutor,
ExtendedEventHandler eventHandler)
throws ProjectResolutionException {
// TODO: b/324127050 - Support other repos.
ImmutableMap<Label, ProjectFilesLookupValue.Key> targetsToSkyKeys =
targets.stream()
.collect(
toImmutableMap(
target -> target,
target -> ProjectFilesLookupValue.key(target.getPackageIdentifier())));
var evalResult =
skyframeExecutor.evaluateSkyKeys(
eventHandler, targetsToSkyKeys.values(), /* keepGoing= */ false);
if (evalResult.hasError()) {
throw new ProjectResolutionException(
"Error finding project files", evalResult.getError().getException());
}
ImmutableMultimap.Builder<Label, Label> ans = ImmutableMultimap.builder();
for (var entry : targetsToSkyKeys.entrySet()) {
ProjectFilesLookupValue containingProjects =
(ProjectFilesLookupValue) evalResult.get(entry.getValue());
ans.putAll(entry.getKey(), containingProjects.getProjectFiles());
}
return ans.build();
}
/**
* Returns the build options to add to this invocation from its active project files and {@code
* --scl_config} setting.
*
* @param fromOptions input {@link BuildOptions}
* @param activeProjects the active project files for this build. An empty {@link Optional} means
* at least one of this build's targets has no project file. If multiple project files are
* active or some targets have project files and others don't, this method checks there's a
* sound way to set the desired config and throws an {@link InvalidConfigurationException} if
* not.
* @param sclConfig the {@link CoreOptions.sclConfig} to apply
* @param allOptionNames the names of every native option the parser recognizes, in {@code "name"}
* form. Not all entries are {@link BuildOptions}.
* @param userOptions options that were set by users (vs. global bazelrcs), in name=value form
* @param configFlagDefinitions definitions of {@code --config=foo} for this build. Null or an
* empty string means use the project-default config if set, otherwise no-op.
* @param enforceCanonicalConfigs if false, project-based flag resolution is disabled
* @param eventHandler handler for non-fatal project-parsing messaging
* @param skyframeExecutor executor for Skyframe evaluation
* @return the options to add to. Caller is responsible for modifying the original build options
* with these additions.
* @throws InvalidConfigurationException if the desired {@code --scl_config} can't be applied in a
* supported way
*/
public static ImmutableSet<String> applySclConfig(
BuildOptions fromOptions,
Project.ActiveProjects activeProjects,
String sclConfig,
ImmutableSet<String> allOptionNames,
ImmutableMap<String, String> userOptions,
ConfigFlagDefinitions configFlagDefinitions,
boolean enforceCanonicalConfigs,
ExtendedEventHandler eventHandler,
SkyframeExecutor skyframeExecutor)
throws InvalidConfigurationException {
var flagSetKeys =
activeProjects.projectFilesToTargetLabels.keySet().stream()
.map(
p ->
FlagSetValue.Key.create(
ImmutableSet.copyOf(activeProjects.projectFilesToTargetLabels.get(p)),
p,
sclConfig,
fromOptions,
allOptionNames,
userOptions,
configFlagDefinitions,
enforceCanonicalConfigs))
.collect(toImmutableSet());
EvaluationResult<SkyValue> result =
skyframeExecutor.evaluateSkyKeys(eventHandler, flagSetKeys, /* keepGoing= */ false);
if (result.hasError()) {
throw new InvalidConfigurationException(
"Cannot parse options: " + result.getError().getException().getMessage(),
Code.INVALID_BUILD_OPTIONS);
}
// Permit multiple configs as long as they all produce the same value, ignoring projects with
// no project files.
ImmutableSet<ImmutableSet<String>> uniqueConfigs =
result.values().stream()
.map(v -> ((FlagSetValue) v).getOptionsFromFlagset())
.collect(toImmutableSet());
if (uniqueConfigs.size() > 1) {
if (!Strings.isNullOrEmpty(sclConfig)) {
throw new InvalidConfigurationException(
"--scl_config=%s resolves to conflicting flagsets: %s"
.formatted(sclConfig, activeProjects.differentProjectsDetails),
Code.INVALID_BUILD_OPTIONS);
}
throw new InvalidConfigurationException(
"Mismatching default configs for a %s. %s"
.formatted(activeProjects.buildType(), activeProjects.differentProjectsDetails),
Code.INVALID_BUILD_OPTIONS);
}
FlagSetValue value = (FlagSetValue) result.values().iterator().next();
value.getPersistentMessages().forEach(eventHandler::handle);
// Options from the selected project config.
return value.getOptionsFromFlagset();
}
}