Delete Ninja support.
The plan was to use it with building Android (the operating system), but that project took another direction and thus parsing Ninja graphs will not be necessary anymore.
If this will ever be needed again, we can resurrect it from source control.
The whole functionality is behind the --experimental_ninja_actions experimental flag, so it can be deleted without warning.
RELNOTES: None.
PiperOrigin-RevId: 447397970
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisAndExecutionResult.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisAndExecutionResult.java
index 9f86899..725c29d 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisAndExecutionResult.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisAndExecutionResult.java
@@ -16,7 +16,6 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedSet;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
@@ -43,8 +42,7 @@
ImmutableSet<ConfiguredTarget> exclusiveTests,
TopLevelArtifactContext topLevelContext,
String workspaceName,
- Collection<TargetAndConfiguration> topLevelTargetsWithConfigs,
- ImmutableSortedSet<String> nonSymlinkedDirectoriesUnderExecRoot) {
+ Collection<TargetAndConfiguration> topLevelTargetsWithConfigs) {
super(
configurations,
targetsToBuild,
@@ -59,7 +57,6 @@
topLevelContext,
/*packageRoots=*/ null,
workspaceName,
- topLevelTargetsWithConfigs,
- nonSymlinkedDirectoriesUnderExecRoot);
+ topLevelTargetsWithConfigs);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java
index a13e10f..12bab8d 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java
@@ -28,7 +28,6 @@
import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.skyframe.SkyFunction;
import net.starlark.java.eval.StarlarkSemantics;
@@ -94,23 +93,6 @@
SpecialArtifact getSymlinkArtifact(PathFragment rootRelativePath, ArtifactRoot root);
/**
- * Creates a source artifact.
- *
- * <p>Do <b>NOT</b> use this unless you have a very good reason to do so and have consulted with
- * someone knowledgeable. Source artifacts should be only created through {@link
- * com.google.devtools.build.lib.analysis.configuredtargets.InputFileConfiguredTarget} by {@link
- * ConfiguredTargetFactory}.
- *
- * <p>This method should only be used when that's not an option. Currently, the only known use
- * case is the {@code ninja_build} rule, which, if it couldn't create source artifacts, would have
- * to have every source artifact that Ninja actions use enumerated in its {@code srcs} attribute.
- *
- * <p>If you use this erroneously, inconsistencies can occur, for example, creating a source
- * artifact with the wrong package path entry or in the wrong package.
- */
- Artifact getSourceArtifactForNinjaBuild(PathFragment execpath, Root root);
-
- /**
* Returns the artifact for the derived file {@code rootRelativePath}, creating it if necessary,
* and setting the root of that artifact to {@code root}. The artifact will represent the output
* directory of a {@code Fileset}.
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisResult.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisResult.java
index 2c5c153..5b3c0c0 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisResult.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisResult.java
@@ -16,7 +16,6 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Sets;
import com.google.devtools.build.lib.actions.ActionGraph;
import com.google.devtools.build.lib.actions.Artifact;
@@ -43,7 +42,6 @@
private final PackageRoots packageRoots;
private final String workspaceName;
private final Collection<TargetAndConfiguration> topLevelTargetsWithConfigs;
- private final ImmutableSortedSet<String> nonSymlinkedDirectoriesUnderExecRoot;
AnalysisResult(
BuildConfigurationCollection configurations,
@@ -59,8 +57,7 @@
TopLevelArtifactContext topLevelContext,
PackageRoots packageRoots,
String workspaceName,
- Collection<TargetAndConfiguration> topLevelTargetsWithConfigs,
- ImmutableSortedSet<String> nonSymlinkedDirectoriesUnderExecRoot) {
+ Collection<TargetAndConfiguration> topLevelTargetsWithConfigs) {
this.configurations = configurations;
this.targetsToBuild = targetsToBuild;
this.aspects = aspects;
@@ -75,7 +72,6 @@
this.packageRoots = packageRoots;
this.workspaceName = workspaceName;
this.topLevelTargetsWithConfigs = topLevelTargetsWithConfigs;
- this.nonSymlinkedDirectoriesUnderExecRoot = nonSymlinkedDirectoriesUnderExecRoot;
}
public BuildConfigurationCollection getConfigurationCollection() {
@@ -159,10 +155,6 @@
return topLevelTargetsWithConfigs;
}
- public ImmutableSortedSet<String> getNonSymlinkedDirectoriesUnderExecRoot() {
- return nonSymlinkedDirectoriesUnderExecRoot;
- }
-
/**
* Returns an equivalent {@link AnalysisResult}, except with exclusive tests treated as parallel
* tests.
@@ -182,7 +174,6 @@
topLevelContext,
packageRoots,
workspaceName,
- topLevelTargetsWithConfigs,
- nonSymlinkedDirectoriesUnderExecRoot);
+ topLevelTargetsWithConfigs);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java
index cbd725b..f66951f 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java
@@ -608,8 +608,7 @@
exclusiveTests,
topLevelOptions,
loadingResult.getWorkspaceName(),
- topLevelTargetsWithConfigs.getTargetsAndConfigs(),
- loadingResult.getNotSymlinkedInExecrootDirectories());
+ topLevelTargetsWithConfigs.getTargetsAndConfigs());
}
@@ -653,8 +652,7 @@
topLevelOptions,
skyframeAnalysisResult.getPackageRoots(),
loadingResult.getWorkspaceName(),
- topLevelTargetsWithConfigs.getTargetsAndConfigs(),
- loadingResult.getNotSymlinkedInExecrootDirectories());
+ topLevelTargetsWithConfigs.getTargetsAndConfigs());
}
/**
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java b/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java
index f43bbdf..f5ab1f8 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java
@@ -38,7 +38,6 @@
import com.google.devtools.build.lib.skyframe.WorkspaceStatusValue;
import com.google.devtools.build.lib.util.Pair;
import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.skyframe.SkyFunction;
import java.io.PrintWriter;
import java.io.StringWriter;
@@ -304,11 +303,6 @@
}
@Override
- public Artifact getSourceArtifactForNinjaBuild(PathFragment execPath, Root root) {
- return artifactFactory.getSourceArtifact(execPath, root, owner);
- }
-
- @Override
public Artifact.DerivedArtifact getFilesetArtifact(
PathFragment rootRelativePath, ArtifactRoot root) {
Preconditions.checkState(enabled);
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD
index 30c9c6e..66d182d 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD
@@ -138,7 +138,6 @@
"//src/main/java/com/google/devtools/build/lib/bazel/repository",
"//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark",
"//src/main/java/com/google/devtools/build/lib/bazel/rules",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions",
"//src/main/java/com/google/devtools/build/lib/buildeventservice",
"//src/main/java/com/google/devtools/build/lib/dynamic",
"//src/main/java/com/google/devtools/build/lib/includescanning",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java
index 5819256..760d64b 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java
@@ -67,7 +67,6 @@
com.google.devtools.build.lib.sandbox.SandboxModule.class,
com.google.devtools.build.lib.runtime.BuildSummaryStatsModule.class,
com.google.devtools.build.lib.dynamic.DynamicExecutionModule.class,
- com.google.devtools.build.lib.bazel.rules.ninja.actions.NinjaRulesModule.class,
com.google.devtools.build.lib.bazel.rules.BazelRulesModule.class,
com.google.devtools.build.lib.bazel.rules.BazelStrategyModule.class,
com.google.devtools.build.lib.network.NoOpConnectivityModule.class,
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/BUILD
index 155478f..5a94a07 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BUILD
@@ -13,7 +13,6 @@
"//src/main/java/com/google/devtools/build/lib/bazel/rules/cpp:srcs",
"//src/main/java/com/google/devtools/build/lib/bazel/rules/genrule:srcs",
"//src/main/java/com/google/devtools/build/lib/bazel/rules/java:srcs",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja:srcs",
"//src/main/java/com/google/devtools/build/lib/bazel/rules/objc:srcs",
"//src/main/java/com/google/devtools/build/lib/bazel/rules/python:srcs",
"//src/main/java/com/google/devtools/build/lib/bazel/rules/sh:srcs",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/BUILD
deleted file mode 100644
index c05c3a8..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/BUILD
+++ /dev/null
@@ -1,15 +0,0 @@
-package(
- default_visibility = ["//src:__subpackages__"],
-)
-
-filegroup(
- name = "srcs",
- srcs = glob(["*"]) + [
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions:srcs",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file:srcs",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer:srcs",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser:srcs",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline:srcs",
- ],
- visibility = ["//src:__subpackages__"],
-)
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/BUILD
deleted file mode 100644
index d5567ca..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/BUILD
+++ /dev/null
@@ -1,50 +0,0 @@
-load("@rules_java//java:defs.bzl", "java_library")
-
-package(
- default_visibility = ["//src:__subpackages__"],
-)
-
-filegroup(
- name = "srcs",
- srcs = glob(["*"]),
- visibility = ["//src:__subpackages__"],
-)
-
-java_library(
- name = "actions",
- srcs = glob(["*.java"]),
- deps = [
- "//src/main/java/com/google/devtools/build/lib:runtime",
- "//src/main/java/com/google/devtools/build/lib/actions",
- "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
- "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
- "//src/main/java/com/google/devtools/build/lib/analysis:actions/symlink_action",
- "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
- "//src/main/java/com/google/devtools/build/lib/analysis:configured_target",
- "//src/main/java/com/google/devtools/build/lib/analysis:file_provider",
- "//src/main/java/com/google/devtools/build/lib/analysis:rule_definition_environment",
- "//src/main/java/com/google/devtools/build/lib/analysis:transitive_info_collection",
- "//src/main/java/com/google/devtools/build/lib/analysis:transitive_info_provider",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser:parser_impl",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline:pipeline_impl",
- "//src/main/java/com/google/devtools/build/lib/cmdline",
- "//src/main/java/com/google/devtools/build/lib/collect/nestedset",
- "//src/main/java/com/google/devtools/build/lib/concurrent",
- "//src/main/java/com/google/devtools/build/lib/packages",
- "//src/main/java/com/google/devtools/build/lib/packages/semantics",
- "//src/main/java/com/google/devtools/build/lib/rules/cpp",
- "//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster",
- "//src/main/java/com/google/devtools/build/lib/skyframe:track_source_directories_flag",
- "//src/main/java/com/google/devtools/build/lib/util",
- "//src/main/java/com/google/devtools/build/lib/util:filetype",
- "//src/main/java/com/google/devtools/build/lib/vfs",
- "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
- "//src/main/java/com/google/devtools/build/skyframe",
- "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
- "//src/main/java/net/starlark/java/eval",
- "//src/main/protobuf:failure_details_java_proto",
- "//third_party:guava",
- "//third_party:jsr305",
- ],
-)
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaAction.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaAction.java
deleted file mode 100644
index cf650be..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaAction.java
+++ /dev/null
@@ -1,257 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.actions;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
-import com.google.devtools.build.lib.actions.AbstractAction;
-import com.google.devtools.build.lib.actions.ActionCacheAwareAction;
-import com.google.devtools.build.lib.actions.ActionEnvironment;
-import com.google.devtools.build.lib.actions.ActionExecutionContext;
-import com.google.devtools.build.lib.actions.ActionOwner;
-import com.google.devtools.build.lib.actions.Artifact;
-import com.google.devtools.build.lib.actions.ArtifactResolver;
-import com.google.devtools.build.lib.actions.ArtifactRoot;
-import com.google.devtools.build.lib.actions.CommandLines;
-import com.google.devtools.build.lib.actions.CommandLines.CommandLineLimits;
-import com.google.devtools.build.lib.actions.EnvironmentalExecException;
-import com.google.devtools.build.lib.actions.ExecException;
-import com.google.devtools.build.lib.actions.RunfilesSupplier;
-import com.google.devtools.build.lib.actions.SpawnResult;
-import com.google.devtools.build.lib.analysis.actions.SpawnAction;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget;
-import com.google.devtools.build.lib.cmdline.PackageIdentifier;
-import com.google.devtools.build.lib.cmdline.RepositoryName;
-import com.google.devtools.build.lib.collect.nestedset.NestedSet;
-import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
-import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
-import com.google.devtools.build.lib.rules.cpp.CppIncludeExtractionContext;
-import com.google.devtools.build.lib.server.FailureDetails;
-import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
-import com.google.devtools.build.lib.server.FailureDetails.NinjaAction.Code;
-import com.google.devtools.build.lib.skyframe.TrackSourceDirectoriesFlag;
-import com.google.devtools.build.lib.util.DependencySet;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.build.lib.vfs.Root;
-import java.io.IOException;
-import java.util.List;
-import javax.annotation.Nullable;
-
-/** Generic class for Ninja actions. Corresponds to the {@link NinjaTarget} in the Ninja file. */
-public class NinjaAction extends SpawnAction implements ActionCacheAwareAction {
- private static final String MNEMONIC = "NinjaGenericAction";
-
- private final Root sourceRoot;
- @Nullable private final Artifact depFile;
- private final ImmutableMap<PathFragment, Artifact> allowedDerivedInputs;
- private final ArtifactRoot derivedOutputRoot;
- private final NestedSet<Artifact> originalInputs; // This does not include order-only inputs.
- private final NestedSet<Artifact> orderOnlyInputs;
-
- public NinjaAction(
- ActionOwner owner,
- Root sourceRoot,
- NestedSet<Artifact> tools,
- NestedSet<Artifact> inputs,
- NestedSet<Artifact> orderOnlyInputs,
- List<? extends Artifact> outputs,
- CommandLines commandLines,
- ActionEnvironment env,
- ImmutableMap<String, String> executionInfo,
- CharSequence progressMessage,
- RunfilesSupplier runfilesSupplier,
- boolean executeUnconditionally,
- @Nullable Artifact depFile,
- ArtifactRoot derivedOutputRoot) {
- super(
- /* owner= */ owner,
- /* tools= */ tools,
- /* inputs= */ inputs,
- /* outputs= */ outputs,
- /* primaryOutput= */ Iterables.getFirst(outputs, null),
- /* resourceSetOrBuilder= */ AbstractAction.DEFAULT_RESOURCE_SET,
- /* commandLines= */ commandLines,
- /* commandLineLimits= */ CommandLineLimits.UNLIMITED,
- /* isShellCommand= */ true,
- /* env= */ env,
- /* executionInfo= */ executionInfo,
- /* progressMessage= */ progressMessage,
- /* runfilesSupplier= */ runfilesSupplier,
- /* mnemonic= */ MNEMONIC,
- /* executeUnconditionally= */ executeUnconditionally,
- /* extraActionInfoSupplier= */ null,
- /* resultConsumer= */ null,
- /*stripOutputPaths=*/ false);
- this.sourceRoot = sourceRoot;
- this.depFile = depFile;
-
- ImmutableMap.Builder<PathFragment, Artifact> allowedDerivedInputsBuilder =
- ImmutableMap.builder();
- for (Artifact input : inputs.toList()) {
- if (!input.isSourceArtifact()) {
- allowedDerivedInputsBuilder.put(input.getExecPath(), input);
- }
- }
- for (Artifact input : orderOnlyInputs.toList()) {
- if (!input.isSourceArtifact()) {
- allowedDerivedInputsBuilder.put(input.getExecPath(), input);
- }
- }
- this.allowedDerivedInputs = allowedDerivedInputsBuilder.buildOrThrow();
-
- this.derivedOutputRoot = derivedOutputRoot;
- this.originalInputs = inputs;
- this.orderOnlyInputs = orderOnlyInputs;
- }
-
- @Override
- protected void beforeExecute(ActionExecutionContext actionExecutionContext) throws ExecException {
- if (!TrackSourceDirectoriesFlag.trackSourceDirectories()) {
- checkInputsForDirectories(
- actionExecutionContext.getEventHandler(), actionExecutionContext.getMetadataProvider());
- }
- }
-
- @Override
- protected void afterExecute(
- ActionExecutionContext actionExecutionContext, List<SpawnResult> spawnResults)
- throws EnvironmentalExecException {
- checkOutputsForDirectories(actionExecutionContext);
-
- if (depFile == null) {
- // The inputs may have been modified during input discovery to include order-only inputs.
- // Restore the original inputs to exclude those order-only inputs, so that order-only inputs
- // do not cause the action to be rerun. This needs to be done when there is no dep file
- // because updateInputsFromDepfile() will recalculate all the dependencies, and even if an
- // input is order-only, it will cause a rebuild if it's listed in the depfile.
- // Note also that this must check if the order-only inputs are empty, because if they are,
- // discoversInputs() will return false, causing updateInputs() to throw an error.
- if (!orderOnlyInputs.isEmpty()) {
- updateInputs(originalInputs);
- }
- } else {
- updateInputsFromDepfile(actionExecutionContext);
- }
- }
-
- private void updateInputsFromDepfile(ActionExecutionContext actionExecutionContext)
- throws EnvironmentalExecException {
-
- boolean siblingRepositoryLayout =
- actionExecutionContext
- .getOptions()
- .getOptions(BuildLanguageOptions.class)
- .experimentalSiblingRepositoryLayout;
-
- CppIncludeExtractionContext scanningContext =
- actionExecutionContext.getContext(CppIncludeExtractionContext.class);
- ArtifactResolver artifactResolver = scanningContext.getArtifactResolver();
- Path execRoot = actionExecutionContext.getExecRoot();
- try {
- DependencySet depSet =
- new DependencySet(execRoot).read(actionExecutionContext.getInputPath(depFile));
- NestedSetBuilder<Artifact> inputsBuilder = NestedSetBuilder.stableOrder();
- for (Path inputPath : depSet.getDependencies()) {
- PathFragment execRelativePath;
-
- // This branch needed in case the depfile contains an absolute path to a source file.
- if (sourceRoot.contains(inputPath)) {
- execRelativePath = inputPath.asFragment().relativeTo(sourceRoot.asPath().asFragment());
- } else {
- execRelativePath = inputPath.asFragment().relativeTo(execRoot.asFragment());
- }
-
- Artifact inputArtifact = null;
- if (allowedDerivedInputs.containsKey(execRelativePath)) {
- // Predeclared generated input.
- inputArtifact = allowedDerivedInputs.get(execRelativePath);
- }
- if (inputArtifact == null) {
- RepositoryName repository =
- PackageIdentifier.discoverFromExecPath(
- execRelativePath, false, siblingRepositoryLayout)
- .getRepository();
- // This check is important because files generated by Ninja are not under the regular
- // derived roots, so resolveSourceArtifact() would happily return a "source artifact"
- // for them
- if (!execRelativePath.startsWith(derivedOutputRoot.getExecPath())) {
- inputArtifact = artifactResolver.resolveSourceArtifact(execRelativePath, repository);
- }
- }
-
- if (inputArtifact == null) {
- throw new EnvironmentalExecException(
- createFailureDetail(
- String.format(
- "depfile-declared dependency '%s' is invalid: it must either be "
- + "a source input, or a pre-declared generated input",
- execRelativePath),
- Code.INVALID_DEPFILE_DECLARED_DEPENDENCY));
- }
-
- inputsBuilder.add(inputArtifact);
- }
- updateInputs(inputsBuilder.build());
- } catch (IOException e) {
- // Some kind of IO or parse exception--wrap & rethrow it to stop the build.
- String message = "error while parsing .d file: " + e.getMessage();
- throw new EnvironmentalExecException(
- e, createFailureDetail(message, Code.D_FILE_PARSE_FAILURE));
- }
- }
-
- @Override
- public boolean discoversInputs() {
- return depFile != null || !orderOnlyInputs.isEmpty();
- }
-
- @Override
- public NestedSet<Artifact> getAllowedDerivedInputs() {
- return getInputs();
- }
-
- @Override
- public NestedSet<Artifact> discoverInputs(ActionExecutionContext actionExecutionContext) {
- NestedSet<Artifact> inputs =
- NestedSetBuilder.<Artifact>stableOrder()
- .addTransitive(getInputs())
- .addTransitive(orderOnlyInputs)
- .build();
- updateInputs(inputs);
- return inputs;
- }
-
- @Override
- protected NestedSet<Artifact> getOriginalInputs() {
- return originalInputs;
- }
-
- private static FailureDetail createFailureDetail(String message, Code detailedCode) {
- return FailureDetail.newBuilder()
- .setMessage(message)
- .setNinjaAction(FailureDetails.NinjaAction.newBuilder().setCode(detailedCode))
- .build();
- }
-
- /**
- * NinjaAction relies on the action cache entry's file list to avoid re-running input discovery
- * after a shutdown.
- */
- @Override
- public boolean storeInputsExecPathsInActionCache() {
- return discoversInputs();
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaActionsHelper.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaActionsHelper.java
deleted file mode 100644
index 77bae44..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaActionsHelper.java
+++ /dev/null
@@ -1,334 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.actions;
-
-import static java.util.stream.Collectors.joining;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Sets;
-import com.google.devtools.build.lib.actions.Artifact;
-import com.google.devtools.build.lib.actions.Artifact.DerivedArtifact;
-import com.google.devtools.build.lib.actions.CommandLines;
-import com.google.devtools.build.lib.actions.EmptyRunfilesSupplier;
-import com.google.devtools.build.lib.analysis.OutputGroupInfo;
-import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
-import com.google.devtools.build.lib.analysis.RuleContext;
-import com.google.devtools.build.lib.analysis.ShToolchain;
-import com.google.devtools.build.lib.analysis.actions.FileWriteAction;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaRuleVariable;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget.InputKind;
-import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
-import com.google.devtools.build.lib.collect.nestedset.Order;
-import com.google.devtools.build.lib.packages.TargetUtils;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Consumer;
-import javax.annotation.Nullable;
-
-/**
- * Helper class for creating {@link NinjaAction} for each {@link NinjaTarget}, and linking the file
- * under the output_root into corresponding directory in execroot. See output_root property in the
- * {@link NinjaGraphRule}.
- */
-public class NinjaActionsHelper {
-
- private final RuleContext ruleContext;
- private final RuleConfiguredTargetBuilder ruleConfiguredTargetBuilder;
- private final ImmutableSortedMap<PathFragment, NinjaTarget> targetsMap;
- private final ImmutableSortedMap<PathFragment, PhonyTarget> phonyTargets;
-
- private final NinjaGraphArtifactsHelper artifactsHelper;
-
- private final PathFragment shellExecutable;
- private final ImmutableSortedMap<String, String> executionInfo;
- private final PhonyTargetArtifacts phonyTargetArtifacts;
- private final List<PathFragment> pathsToBuild;
- private final ImmutableSet<PathFragment> outputRootInputsSymlinks;
-
- /**
- * Constructor
- *
- * @param ruleContext parent NinjaGraphRule rule context
- * @param artifactsHelper helper object to create artifacts
- * @param targetsMap mapping of outputs to all non-phony Ninja targets from Ninja file
- * @param phonyTargets mapping of names to all phony Ninja actions from Ninja file
- * @param phonyTargetArtifacts helper class for computing transitively included artifacts of phony
- * targets
- * @param pathsToBuild paths requested by the user to be build (in output_groups attribute)
- */
- NinjaActionsHelper(
- RuleContext ruleContext,
- RuleConfiguredTargetBuilder ruleConfiguredTargetBuilder,
- NinjaGraphArtifactsHelper artifactsHelper,
- ImmutableSortedMap<PathFragment, NinjaTarget> targetsMap,
- ImmutableSortedMap<PathFragment, PhonyTarget> phonyTargets,
- PhonyTargetArtifacts phonyTargetArtifacts,
- List<PathFragment> pathsToBuild,
- ImmutableSet<PathFragment> outputRootInputsSymlinks) {
- this.ruleContext = ruleContext;
- this.artifactsHelper = artifactsHelper;
- this.targetsMap = targetsMap;
- this.phonyTargets = phonyTargets;
- this.shellExecutable = ShToolchain.getPathOrError(ruleContext);
- this.executionInfo = createExecutionInfo(ruleContext);
- this.phonyTargetArtifacts = phonyTargetArtifacts;
- this.pathsToBuild = pathsToBuild;
- this.outputRootInputsSymlinks = outputRootInputsSymlinks;
- this.ruleConfiguredTargetBuilder = ruleConfiguredTargetBuilder;
- }
-
- void createNinjaActions() throws GenericParsingException {
- // Traverse the action graph starting from the targets, specified by the user.
- // Only create the required actions.
- Set<PathFragment> visitedPaths = Sets.newHashSet();
- Set<NinjaTarget> visitedTargets = Sets.newHashSet();
- visitedPaths.addAll(pathsToBuild);
- ArrayDeque<PathFragment> queue = new ArrayDeque<>(pathsToBuild);
- Consumer<Collection<PathFragment>> enqueuer =
- paths -> {
- for (PathFragment input : paths) {
- if (visitedPaths.add(input)) {
- queue.add(input);
- }
- }
- };
- while (!queue.isEmpty()) {
- PathFragment fragment = queue.remove();
- NinjaTarget target = targetsMap.get(fragment);
- if (target != null) {
- // If the output is already created by a symlink action created from specifying that
- // file in output_root_inputs attribute of the ninja_graph rule, do not create other
- // actions that output that same file, since that will result in an action conflict.
- if (!outputRootInputsSymlinks.contains(fragment)) {
- if (visitedTargets.add(target)) {
- createNinjaAction(target);
- }
- enqueuer.accept(target.getAllInputs());
- } else {
- // Verify that the Ninja action we're skipping (because its outputs are already
- // being symlinked using output_root_inputs) has only symlink outputs specified in
- // output_root_inputs. Otherwise we might skip some other outputs.
- List<PathFragment> outputsInOutputRootInputsSymlinks = new ArrayList<>();
- List<PathFragment> outputsNotInOutputRootInputsSymlinks = new ArrayList<>();
- for (PathFragment output : target.getAllOutputs()) {
- if (outputRootInputsSymlinks.contains(output)) {
- outputsInOutputRootInputsSymlinks.add(output);
- } else {
- outputsNotInOutputRootInputsSymlinks.add(output);
- }
- }
- if (!outputsNotInOutputRootInputsSymlinks.isEmpty()) {
- throw new GenericParsingException(
- "Ninja target "
- + target.getRuleName()
- + " has "
- + "outputs in output_root_inputs and other outputs not in output_root_inputs:\n"
- + "Outputs in output_root_inputs:\n "
- + Joiner.on(" \n").join(outputsInOutputRootInputsSymlinks)
- + "\nOutputs not in output_root_inputs:\n "
- + Joiner.on(" \n").join(outputsNotInOutputRootInputsSymlinks));
- }
- }
- } else {
- PhonyTarget phonyTarget = phonyTargets.get(fragment);
- // Phony target can be null, if the path in neither regular or phony target,
- // but the source file.
- if (phonyTarget != null) {
- phonyTarget.visitExplicitInputs(phonyTargets, enqueuer::accept);
- }
- }
- }
- }
-
- private void createNinjaAction(NinjaTarget target) throws GenericParsingException {
- NestedSetBuilder<Artifact> inputsBuilder = NestedSetBuilder.stableOrder();
- NestedSetBuilder<Artifact> orderOnlyInputsBuilder = NestedSetBuilder.stableOrder();
- NestedSetBuilder<Artifact> validationInputs = NestedSetBuilder.stableOrder();
- ImmutableList.Builder<Artifact> outputsBuilder = ImmutableList.builder();
- boolean isAlwaysDirty =
- fillArtifacts(
- target, inputsBuilder, orderOnlyInputsBuilder, validationInputs, outputsBuilder);
-
- ImmutableSortedMap<NinjaRuleVariable, String> resolvedMap = target.computeRuleVariables();
- String command = resolvedMap.get(NinjaRuleVariable.COMMAND);
- maybeCreateRspFile(target.getRuleName(), inputsBuilder, resolvedMap);
-
- if (!artifactsHelper.getWorkingDirectory().isEmpty()) {
- command = String.format("cd %s && ", artifactsHelper.getWorkingDirectory()) + command;
- }
- CommandLines commandLines =
- CommandLines.of(ImmutableList.of(shellExecutable.getPathString(), "-c", command));
- Artifact depFile = getDepfile(resolvedMap);
- if (depFile != null) {
- outputsBuilder.add(depFile);
- }
-
- List<Artifact> outputs = outputsBuilder.build();
-
- if (!validationInputs.isEmpty()) {
- ruleConfiguredTargetBuilder.addOutputGroup(
- OutputGroupInfo.VALIDATION, validationInputs.build());
- }
-
- ruleContext.registerAction(
- new NinjaAction(
- ruleContext.getActionOwner(),
- artifactsHelper.getSourceRoot(),
- NestedSetBuilder.emptySet(Order.STABLE_ORDER),
- inputsBuilder.build(),
- orderOnlyInputsBuilder.build(),
- outputs,
- commandLines,
- Preconditions.checkNotNull(ruleContext.getConfiguration()).getActionEnvironment(),
- executionInfo,
- createProgressMessage(target, resolvedMap, outputs),
- EmptyRunfilesSupplier.INSTANCE,
- isAlwaysDirty,
- depFile,
- artifactsHelper.getDerivedOutputRoot()));
- }
-
- /**
- * Create a progress message for the ninja action.
- *
- * <ul>
- * <li>If the target has a "description" variable, use that. It has been expanded at parse time
- * with file variables.
- * <li>If the rule for the target has a description, use that. It has been expanded with rule,
- * build and file variables.
- * <li>Else, generate a pretty-printed progress message at runtime, using the rule name and
- * output filenames for a general idea on what the action is doing, without printing the
- * full command line (which can be surfaced with --subcommands, anyway).
- */
- private static String createProgressMessage(
- NinjaTarget target,
- ImmutableSortedMap<NinjaRuleVariable, String> ruleVariables,
- List<Artifact> outputs) {
- String ruleDescription = ruleVariables.get(NinjaRuleVariable.DESCRIPTION);
- if (ruleDescription != null) {
- return ruleDescription;
- }
-
- String ruleName = target.getRuleName();
- StringBuilder messageBuilder = new StringBuilder();
- if (!ruleName.isEmpty()) {
- messageBuilder.append("[rule ").append(ruleName).append("] ");
- }
- messageBuilder.append("Outputs: ");
- messageBuilder.append(outputs.stream().map(Artifact::getFilename).collect(joining(", ")));
- return messageBuilder.toString();
- }
-
- /** Returns true if the action should be marked as always dirty. */
- private boolean fillArtifacts(
- NinjaTarget target,
- NestedSetBuilder<Artifact> inputsBuilder,
- NestedSetBuilder<Artifact> orderOnlyInputsBuilder,
- NestedSetBuilder<Artifact> validationInputsBuilder,
- ImmutableList.Builder<Artifact> outputsBuilder)
- throws GenericParsingException {
-
- boolean isAlwaysDirty = false;
- for (Map.Entry<InputKind, PathFragment> entry : target.getAllInputsAndKind()) {
-
- InputKind kind = entry.getKey();
- NestedSetBuilder<Artifact> builder;
- if (kind == InputKind.ORDER_ONLY) {
- builder = orderOnlyInputsBuilder;
- } else if (kind == InputKind.VALIDATION) {
- // Note that validation inputs are specific to AOSP's Ninja implementation.
- builder = validationInputsBuilder;
- } else {
- builder = inputsBuilder;
- }
-
- PathFragment input = entry.getValue();
- PhonyTarget phonyTarget = this.phonyTargets.get(input);
- if (phonyTarget != null) {
- builder.addTransitive(phonyTargetArtifacts.getPhonyTargetArtifacts(input));
- isAlwaysDirty |= (phonyTarget.isAlwaysDirty() && kind != InputKind.ORDER_ONLY);
- } else {
- Artifact artifact = artifactsHelper.getInputArtifact(input);
- builder.add(artifact);
- }
- }
-
- for (PathFragment output : target.getAllOutputs()) {
- outputsBuilder.add(artifactsHelper.createOutputArtifact(output));
- }
- return isAlwaysDirty;
- }
-
- @Nullable
- private Artifact getDepfile(ImmutableSortedMap<NinjaRuleVariable, String> ruleVariables)
- throws GenericParsingException {
- String depfileName = ruleVariables.get(NinjaRuleVariable.DEPFILE);
- if (depfileName != null) {
- if (!depfileName.trim().isEmpty()) {
- return artifactsHelper.createOutputArtifact(PathFragment.create(depfileName));
- }
- }
- return null;
- }
-
- private void maybeCreateRspFile(
- String ruleName,
- NestedSetBuilder<Artifact> inputsBuilder,
- ImmutableSortedMap<NinjaRuleVariable, String> ruleVariables)
- throws GenericParsingException {
- String fileName = ruleVariables.get(NinjaRuleVariable.RSPFILE);
- String contentString = ruleVariables.get(NinjaRuleVariable.RSPFILE_CONTENT);
-
- if (fileName == null && contentString == null) {
- return;
- }
- if (fileName == null || contentString == null) {
- ruleContext.ruleError(
- String.format(
- "Both rspfile and rspfile_content should be defined for rule '%s'.", ruleName));
- return;
- }
-
- if (!fileName.trim().isEmpty()) {
- DerivedArtifact rspArtifact =
- artifactsHelper.createOutputArtifact(PathFragment.create(fileName));
- FileWriteAction fileWriteAction =
- FileWriteAction.create(ruleContext, rspArtifact, contentString, false);
- ruleContext.registerAction(fileWriteAction);
- inputsBuilder.add(rspArtifact);
- }
- }
-
- private static ImmutableSortedMap<String, String> createExecutionInfo(RuleContext ruleContext) {
- ImmutableSortedMap.Builder<String, String> builder = ImmutableSortedMap.naturalOrder();
- builder.putAll(TargetUtils.getExecutionInfo(ruleContext.getRule()));
- builder.put("local", "");
- ImmutableSortedMap<String, String> map = builder.buildOrThrow();
- Preconditions.checkNotNull(ruleContext.getConfiguration())
- .modifyExecutionInfo(map, "NinjaRule");
- return map;
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaBuild.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaBuild.java
deleted file mode 100644
index fc6a98a..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaBuild.java
+++ /dev/null
@@ -1,233 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.actions;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Maps;
-import com.google.devtools.build.lib.actions.Artifact;
-import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
-import com.google.devtools.build.lib.analysis.ConfiguredTarget;
-import com.google.devtools.build.lib.analysis.FileProvider;
-import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
-import com.google.devtools.build.lib.analysis.RuleConfiguredTargetFactory;
-import com.google.devtools.build.lib.analysis.RuleContext;
-import com.google.devtools.build.lib.analysis.RunfilesProvider;
-import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
-import com.google.devtools.build.lib.analysis.actions.SymlinkAction;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.collect.nestedset.NestedSet;
-import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
-import com.google.devtools.build.lib.collect.nestedset.NestedSetVisitor;
-import com.google.devtools.build.lib.collect.nestedset.NestedSetVisitor.VisitedState;
-import com.google.devtools.build.lib.collect.nestedset.Order;
-import com.google.devtools.build.lib.packages.Type;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.stream.Collectors;
-import javax.annotation.Nullable;
-
-/** Configured target factory for {@link NinjaBuildRule}. */
-public class NinjaBuild implements RuleConfiguredTargetFactory {
-
- @Override
- @Nullable
- public ConfiguredTarget create(RuleContext ruleContext)
- throws InterruptedException, RuleErrorException, ActionConflictException {
- Map<String, List<String>> outputGroupsFromAttrs =
- ruleContext.attributes().get("output_groups", Type.STRING_LIST_DICT);
- NinjaGraphProvider graphProvider =
- ruleContext.getPrerequisite("ninja_graph", NinjaGraphProvider.class);
- Preconditions.checkNotNull(graphProvider);
- List<PathFragment> pathsToBuild =
- outputGroupsFromAttrs.values().stream()
- .flatMap(List::stream)
- .map(PathFragment::create)
- .collect(Collectors.toList());
- ImmutableSortedMap.Builder<PathFragment, Artifact> depsMapBuilder =
- ImmutableSortedMap.naturalOrder();
- ImmutableSortedMap.Builder<PathFragment, Artifact> symlinksMapBuilder =
- ImmutableSortedMap.naturalOrder();
- createDepsMap(
- ruleContext, graphProvider.getWorkingDirectory(), depsMapBuilder, symlinksMapBuilder);
-
- ImmutableSortedMap<PathFragment, Artifact> depsMap = depsMapBuilder.buildOrThrow();
-
- NinjaGraphArtifactsHelper artifactsHelper =
- new NinjaGraphArtifactsHelper(
- ruleContext,
- graphProvider.getOutputRoot(),
- graphProvider.getWorkingDirectory(),
- symlinksMapBuilder.buildOrThrow(),
- graphProvider.getOutputRootSymlinks());
- if (ruleContext.hasErrors()) {
- return null;
- }
-
- RuleConfiguredTargetBuilder ruleConfiguredTargetBuilder =
- new RuleConfiguredTargetBuilder(ruleContext);
-
- try {
- symlinkDepsMappings(ruleContext, artifactsHelper, depsMap);
-
- PhonyTargetArtifacts phonyTargetArtifacts =
- new PhonyTargetArtifacts(graphProvider.getPhonyTargetsMap(), artifactsHelper);
- ImmutableSet<PathFragment> symlinks =
- ImmutableSet.<PathFragment>builder()
- .addAll(graphProvider.getOutputRootInputsSymlinks())
- .addAll(depsMap.keySet())
- .build();
-
- new NinjaActionsHelper(
- ruleContext,
- ruleConfiguredTargetBuilder,
- artifactsHelper,
- graphProvider.getTargetsMap(),
- graphProvider.getPhonyTargetsMap(),
- phonyTargetArtifacts,
- pathsToBuild,
- symlinks)
- .createNinjaActions();
-
- if (!checkOrphanArtifacts(ruleContext)) {
- return null;
- }
-
- NestedSetBuilder<Artifact> filesToBuild = NestedSetBuilder.stableOrder();
- TreeMap<String, NestedSet<Artifact>> outputGroups = Maps.newTreeMap();
- for (Map.Entry<String, List<String>> entry : outputGroupsFromAttrs.entrySet()) {
- NestedSet<Artifact> artifacts =
- getGroupArtifacts(
- ruleContext,
- entry.getValue(),
- graphProvider.getPhonyTargetsMap(),
- phonyTargetArtifacts,
- artifactsHelper);
- outputGroups.put(entry.getKey(), artifacts);
- filesToBuild.addTransitive(artifacts);
- }
-
- if (ruleContext.hasErrors()) {
- return null;
- }
-
- return ruleConfiguredTargetBuilder
- .addProvider(RunfilesProvider.class, RunfilesProvider.EMPTY)
- .setFilesToBuild(filesToBuild.build())
- .addOutputGroups(outputGroups)
- .build();
- } catch (GenericParsingException e) {
- ruleContext.ruleError(e.getMessage());
- return null;
- }
- }
-
- private static void symlinkDepsMappings(
- RuleContext ruleContext,
- NinjaGraphArtifactsHelper artifactsHelper,
- ImmutableSortedMap<PathFragment, Artifact> depsMap)
- throws GenericParsingException {
- for (Map.Entry<PathFragment, Artifact> entry : depsMap.entrySet()) {
- PathFragment depPath = entry.getKey();
- Artifact destinationArtifact = entry.getValue();
- Artifact outputArtifact = artifactsHelper.createOutputArtifact(depPath);
-
- SymlinkAction symlinkAction =
- SymlinkAction.toArtifact(
- ruleContext.getActionOwner(),
- destinationArtifact,
- outputArtifact,
- String.format(
- "Symlinking deps_mapping entry '%s' to '%s'",
- destinationArtifact.getExecPath(), outputArtifact.getExecPath()));
- ruleContext.registerAction(symlinkAction);
- }
- }
-
- private static boolean checkOrphanArtifacts(RuleContext ruleContext) {
- ImmutableSet<Artifact> orphanArtifacts =
- ruleContext.getAnalysisEnvironment().getOrphanArtifacts();
- if (!orphanArtifacts.isEmpty()) {
- List<String> paths =
- orphanArtifacts.stream().map(Artifact::getExecPathString).collect(Collectors.toList());
- ruleContext.ruleError(
- "The following artifacts do not have a generating action in Ninja file: "
- + String.join(", ", paths));
- return false;
- }
- return true;
- }
-
- private static NestedSet<Artifact> getGroupArtifacts(
- RuleContext ruleContext,
- List<String> targets,
- ImmutableSortedMap<PathFragment, PhonyTarget> phonyTargetsMap,
- PhonyTargetArtifacts phonyTargetsArtifacts,
- NinjaGraphArtifactsHelper artifactsHelper)
- throws GenericParsingException {
- NestedSetBuilder<Artifact> nestedSetBuilder = NestedSetBuilder.stableOrder();
- for (String target : targets) {
- PathFragment path = PathFragment.create(target);
- if (phonyTargetsMap.containsKey(path)) {
- NestedSet<Artifact> artifacts = phonyTargetsArtifacts.getPhonyTargetArtifacts(path);
- nestedSetBuilder.addTransitive(artifacts);
- } else {
- Artifact outputArtifact = artifactsHelper.createOutputArtifact(path);
- if (outputArtifact == null) {
- ruleContext.ruleError(
- String.format("Required target '%s' is not created in ninja_graph.", path));
- return NestedSetBuilder.emptySet(Order.STABLE_ORDER);
- }
- nestedSetBuilder.add(outputArtifact);
- }
- }
- return nestedSetBuilder.build();
- }
-
- private static void createDepsMap(
- RuleContext ruleContext,
- PathFragment workingDirectory,
- ImmutableSortedMap.Builder<PathFragment, Artifact> depsMapBuilder,
- ImmutableSortedMap.Builder<PathFragment, Artifact> symlinksMapBuilder)
- throws InterruptedException {
- FileProvider fileProvider = ruleContext.getPrerequisite("ninja_graph", FileProvider.class);
- Preconditions.checkNotNull(fileProvider);
- new NestedSetVisitor<Artifact>(
- a -> {
- symlinksMapBuilder.put(a.getExecPath().relativeTo(workingDirectory), a);
- },
- new VisitedState<>())
- .visit(fileProvider.getFilesToBuild());
-
- Map<String, TransitiveInfoCollection> mapping = ruleContext.getPrerequisiteMap("deps_mapping");
- for (Map.Entry<String, TransitiveInfoCollection> entry : mapping.entrySet()) {
- NestedSet<Artifact> filesToBuild =
- entry.getValue().getProvider(FileProvider.class).getFilesToBuild();
- if (!filesToBuild.isSingleton()) {
- ruleContext.attributeError(
- "deps_mapping",
- String.format(
- "'%s' contains more than one output. "
- + "deps_mapping should only contain targets, producing a single output file.",
- entry.getValue().getLabel().getCanonicalForm()));
- return;
- }
- depsMapBuilder.put(PathFragment.create(entry.getKey()), filesToBuild.getSingleton());
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaBuildRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaBuildRule.java
deleted file mode 100644
index e1546ce..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaBuildRule.java
+++ /dev/null
@@ -1,82 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.actions;
-
-import static com.google.devtools.build.lib.packages.Attribute.attr;
-import static com.google.devtools.build.lib.packages.BuildType.LABEL;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.devtools.build.lib.analysis.BaseRuleClasses;
-import com.google.devtools.build.lib.analysis.RuleDefinition;
-import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
-import com.google.devtools.build.lib.packages.BuildType;
-import com.google.devtools.build.lib.packages.RuleClass;
-import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
-import com.google.devtools.build.lib.packages.Type;
-import com.google.devtools.build.lib.util.FileTypeSet;
-
-/**
- * The rule creates the action subgraph from graph of {@link
- * com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget}, parsed by {@link
- * NinjaGraphRule} and passed in the form of {@link NinjaGraphProvider}.
- *
- * <p>The subgraph is computed as all actions needed to build targets from 'output_groups' (phony
- * targets can also be passed there). Bazel-built inputs should be passed with 'deps_mapping'
- * attribute. Currently, if there are two ninja_build targets which refer to intersecting subgraphs
- * in ninja_graph, all the actions will be created by each of ninja_build targets, i.e. duplicates.
- * Bazel will determine that those are duplicates and only execute each action once. Future
- * improvements are planned to avoid creation of duplicate actions, probably with the help of some
- * coordinating registry structure.
- *
- * <p>Currently all input files of the Ninja graph must be in a subdirectory of the package the
- * {@code ninja_build} rule is in. It's currently okay if they are in a subpackage, although that
- * may change later. For best results, put this rule in the top-level BUILD file.
- */
-public class NinjaBuildRule implements RuleDefinition {
- @Override
- public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment environment) {
- return builder
- .add(
- attr("ninja_graph", LABEL)
- .allowedFileTypes(FileTypeSet.ANY_FILE)
- .allowedRuleClasses("ninja_graph")
- .setDoc("ninja_graph that parses all Ninja files that compose a graph of actions."))
- .add(
- attr("deps_mapping", BuildType.LABEL_DICT_UNARY)
- .allowedFileTypes(FileTypeSet.ANY_FILE)
- .setDoc(
- "Mapping of paths in the Ninja file to the Bazel-built dependencies. Main"
- + " output of each dependency will be used as an input to the Ninja"
- + " action which refers to the corresponding path.")
- .value(ImmutableMap.of()))
- .add(
- attr("output_groups", Type.STRING_LIST_DICT)
- .setDoc(
- "Mapping of output groups to the list of output paths in the Ninja file. "
- + "Only the output paths mentioned in this attribute will be built."
- + " Phony target names may be specified as the output paths."))
- .build();
- }
-
- @Override
- public Metadata getMetadata() {
- return RuleDefinition.Metadata.builder()
- .name("ninja_build")
- .type(RuleClassType.NORMAL)
- .ancestors(BaseRuleClasses.NativeBuildRule.class)
- .factoryClass(NinjaBuild.class)
- .build();
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaGraph.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaGraph.java
deleted file mode 100644
index 5cae7cb..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaGraph.java
+++ /dev/null
@@ -1,378 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.actions;
-
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.Iterables;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.devtools.build.lib.actions.Artifact;
-import com.google.devtools.build.lib.actions.Artifact.DerivedArtifact;
-import com.google.devtools.build.lib.actions.FileValue;
-import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
-import com.google.devtools.build.lib.analysis.ConfiguredTarget;
-import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder;
-import com.google.devtools.build.lib.analysis.RuleConfiguredTargetFactory;
-import com.google.devtools.build.lib.analysis.RuleContext;
-import com.google.devtools.build.lib.analysis.RunfilesProvider;
-import com.google.devtools.build.lib.analysis.actions.SymlinkAction;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget;
-import com.google.devtools.build.lib.bazel.rules.ninja.pipeline.NinjaPipelineImpl;
-import com.google.devtools.build.lib.collect.nestedset.NestedSet;
-import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
-import com.google.devtools.build.lib.collect.nestedset.Order;
-import com.google.devtools.build.lib.concurrent.ExecutorUtil;
-import com.google.devtools.build.lib.packages.Type;
-import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
-import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.build.lib.vfs.Root;
-import com.google.devtools.build.lib.vfs.RootedPath;
-import com.google.devtools.build.skyframe.SkyFunction.Environment;
-import com.google.devtools.build.skyframe.SkyKey;
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.stream.Collectors;
-import javax.annotation.Nullable;
-
-/** Configured target factory for {@link NinjaGraphRule}. */
-public class NinjaGraph implements RuleConfiguredTargetFactory {
- /**
- * Use a thread pool with the heuristically determined maximum number of threads, but the policy
- * to not consume threads when there are no active tasks.
- */
- private static final ThreadPoolExecutor NINJA_POOL =
- ExecutorUtil.newSlackPool(
- Math.max(2, Runtime.getRuntime().availableProcessors() / 2),
- NinjaGraph.class.getSimpleName());
-
- @Override
- @Nullable
- public ConfiguredTarget create(RuleContext ruleContext)
- throws InterruptedException, RuleErrorException, ActionConflictException {
- if (!ruleContext
- .getAnalysisEnvironment()
- .getStarlarkSemantics()
- .getBool(BuildLanguageOptions.EXPERIMENTAL_NINJA_ACTIONS)) {
- throw ruleContext.throwWithRuleError(
- "Usage of ninja_graph is only allowed with --experimental_ninja_actions flag");
- }
- Artifact mainArtifact = ruleContext.getPrerequisiteArtifact("main");
- ImmutableList<Artifact> ninjaSrcs = ruleContext.getPrerequisiteArtifacts("ninja_srcs").list();
- PathFragment outputRoot =
- PathFragment.create(ruleContext.attributes().get("output_root", Type.STRING));
- PathFragment workingDirectory =
- PathFragment.create(ruleContext.attributes().get("working_directory", Type.STRING));
- Environment env = ruleContext.getAnalysisEnvironment().getSkyframeEnv();
- establishDependencyOnNinjaFiles(env, mainArtifact, ninjaSrcs);
- checkDirectoriesAttributes(ruleContext, outputRoot, workingDirectory);
-
- if (env.valuesMissing() || ruleContext.hasErrors()) {
- return null;
- }
-
- Root sourceRoot = mainArtifact.getRoot().getRoot();
- NinjaGraphArtifactsHelper artifactsHelper =
- new NinjaGraphArtifactsHelper(
- ruleContext,
- outputRoot,
- workingDirectory,
- ImmutableSortedMap.of(),
- ImmutableSortedSet.of());
- if (ruleContext.hasErrors()) {
- return null;
- }
-
- ImmutableSet<PathFragment> outputRootInputs =
- ruleContext.attributes().get("output_root_inputs", Type.STRING_LIST).stream()
- .map(s -> workingDirectoryRelativePath(artifactsHelper, s))
- .collect(toImmutableSet());
- ImmutableSet<PathFragment> outputRootInputDirs =
- ruleContext.attributes().get("output_root_input_dirs", Type.STRING_LIST).stream()
- .map(s -> workingDirectoryRelativePath(artifactsHelper, s))
- .collect(toImmutableSet());
-
- try {
- TargetsPreparer targetsPreparer = new TargetsPreparer();
- List<Path> childNinjaFiles =
- ninjaSrcs.stream().map(Artifact::getPath).collect(Collectors.toList());
- Path workspace =
- Preconditions.checkNotNull(ruleContext.getConfiguration())
- .getDirectories()
- .getWorkspace();
- String ownerTargetName = ruleContext.getLabel().getName();
- List<NinjaTarget> ninjaTargets =
- new NinjaPipelineImpl(
- workspace.getRelative(workingDirectory),
- MoreExecutors.listeningDecorator(NINJA_POOL),
- childNinjaFiles,
- ownerTargetName)
- .pipeline(mainArtifact.getPath());
- targetsPreparer.prepareTargets(ninjaTargets, outputRootInputs, outputRootInputDirs);
-
- NestedSet<Artifact> outputRootInputsSymlinks =
- createSymlinkActions(
- ruleContext,
- sourceRoot,
- targetsPreparer.getConfirmedOutputRootInputs(),
- artifactsHelper);
- if (ruleContext.hasErrors()) {
- return null;
- }
-
- ImmutableSet<PathFragment> outputRootInputsSymlinksPathFragments =
- outputRootInputsSymlinks.toList().stream()
- .map(Artifact::getExecPath)
- .collect(toImmutableSet());
-
- NinjaGraphProvider ninjaGraphProvider =
- new NinjaGraphProvider(
- outputRoot,
- workingDirectory,
- targetsPreparer.getTargetsMap(),
- targetsPreparer.getPhonyTargetsMap(),
- targetsPreparer.getSymlinkOutputs(),
- outputRootInputsSymlinksPathFragments);
-
- return new RuleConfiguredTargetBuilder(ruleContext)
- .addProvider(RunfilesProvider.class, RunfilesProvider.EMPTY)
- .addProvider(NinjaGraphProvider.class, ninjaGraphProvider)
- .setFilesToBuild(outputRootInputsSymlinks)
- .build();
- } catch (GenericParsingException | IOException e) {
- // IOException is possible with reading Ninja file, describing the action graph.
- ruleContext.ruleError(e.getMessage());
- return null;
- }
- }
-
- /**
- * Given a path string, returns a path fragment representing the path relative to the working
- * directory.
- */
- private PathFragment workingDirectoryRelativePath(
- NinjaGraphArtifactsHelper artifactsHelper, String pathString) {
- return artifactsHelper
- .getOutputRootPath()
- .getRelative(pathString)
- .relativeTo(artifactsHelper.getWorkingDirectory());
- }
-
- private static NestedSet<Artifact> createSymlinkActions(
- RuleContext ruleContext,
- Root sourceRoot,
- ImmutableSet<PathFragment> outputRootInputs,
- NinjaGraphArtifactsHelper artifactsHelper)
- throws GenericParsingException {
-
- if (outputRootInputs.isEmpty()) {
- return NestedSetBuilder.emptySet(Order.STABLE_ORDER);
- }
-
- NestedSetBuilder<Artifact> symlinks = NestedSetBuilder.stableOrder();
-
- Path workingDirectoryRoot =
- Preconditions.checkNotNull(sourceRoot.asPath())
- .getRelative(artifactsHelper.getWorkingDirectory());
-
- for (PathFragment input : outputRootInputs) {
- // output_root_inputs are relative to the output_root directory, and we should
- // pass inside createOutputArtifact() paths, relative to working directory.
- DerivedArtifact derivedArtifact = artifactsHelper.createOutputArtifact(input);
- symlinks.add(derivedArtifact);
-
- PathFragment absolutePath = workingDirectoryRoot.getRelative(input).asFragment();
-
- SymlinkAction symlinkAction =
- SymlinkAction.toAbsolutePath(
- ruleContext.getActionOwner(),
- absolutePath,
- derivedArtifact,
- String.format(
- "Symlinking %s under <execroot>/%s", input, artifactsHelper.getOutputRootPath()));
- ruleContext.registerAction(symlinkAction);
- }
- return symlinks.build();
- }
-
- private static class TargetsPreparer {
- private ImmutableSortedMap<PathFragment, NinjaTarget> targetsMap;
- private ImmutableSortedMap<PathFragment, PhonyTarget> phonyTargetsMap;
- private ImmutableSet<PathFragment> symlinkOutputs;
- // Path fragments of files which are inputs of some NinjaTarget, and also classified as
- // an "output root input". Such files must be symlinked from the output root separately.
- private ImmutableSet<PathFragment> confirmedOutputRootInputs;
-
- public ImmutableSortedMap<PathFragment, NinjaTarget> getTargetsMap() {
- return targetsMap;
- }
-
- public ImmutableSortedMap<PathFragment, PhonyTarget> getPhonyTargetsMap() {
- return phonyTargetsMap;
- }
-
- public ImmutableSet<PathFragment> getSymlinkOutputs() {
- return symlinkOutputs;
- }
-
- public ImmutableSet<PathFragment> getConfirmedOutputRootInputs() {
- return confirmedOutputRootInputs;
- }
-
- void prepareTargets(
- List<NinjaTarget> ninjaTargets,
- ImmutableSet<PathFragment> outputRootInputs,
- ImmutableSet<PathFragment> outputRootInputDirs)
- throws GenericParsingException {
- ImmutableSortedMap.Builder<PathFragment, NinjaTarget> targetsMapBuilder =
- ImmutableSortedMap.naturalOrder();
- ImmutableSortedMap.Builder<PathFragment, NinjaTarget> phonyTargetsBuilder =
- ImmutableSortedMap.naturalOrder();
- ImmutableSet.Builder<PathFragment> symlinkOutputsBuilder = ImmutableSet.builder();
- ImmutableSet.Builder<PathFragment> outputRootInputsBuilder = ImmutableSet.builder();
- categorizeTargetsAndOutputs(
- ninjaTargets,
- outputRootInputs,
- outputRootInputDirs,
- targetsMapBuilder,
- phonyTargetsBuilder,
- symlinkOutputsBuilder,
- outputRootInputsBuilder);
- targetsMap = targetsMapBuilder.buildOrThrow();
- phonyTargetsMap = NinjaPhonyTargetsUtil.getPhonyPathsMap(phonyTargetsBuilder.buildOrThrow());
- symlinkOutputs = symlinkOutputsBuilder.build();
- confirmedOutputRootInputs = outputRootInputsBuilder.build();
- }
-
- /**
- * Iterate over all parsed Ninja targets into phony and non-phony targets in a single pass, and
- * run validations along the way. For non-phony targets, also extract the symlink_outputs for
- * registering symlink artifacts in actions later on, in ninja_build.
- *
- * @param ninjaTargets list of all parsed Ninja targets
- * @param outputRootInputs list of paths under the output root which may be valid input files
- * and thus may be added to {@code outputRootInputsBuilder}
- * @param outputRootInputs list of directory paths under the output root which may contain valid
- * input files to be added to {@code outputRootInputsBuilder}
- * @param targetsBuilder builder for map of path fragments to the non-phony targets
- * @param phonyTargetsBuilder builder for map of path fragments to the phony targets
- * @param symlinkOutputsBuilder builder for set of declared symlink outputs
- * @param symlinkOutputsBuilder builder for set of input files under the output root directory
- */
- private static void categorizeTargetsAndOutputs(
- List<NinjaTarget> ninjaTargets,
- ImmutableSet<PathFragment> outputRootInputs,
- ImmutableSet<PathFragment> outputRootInputDirs,
- ImmutableSortedMap.Builder<PathFragment, NinjaTarget> targetsBuilder,
- ImmutableSortedMap.Builder<PathFragment, NinjaTarget> phonyTargetsBuilder,
- ImmutableSet.Builder<PathFragment> symlinkOutputsBuilder,
- ImmutableSet.Builder<PathFragment> outputRootInputsBuilder)
- throws GenericParsingException {
- for (NinjaTarget target : ninjaTargets) {
- // Add input path to outputRootInputs if either it's listed in outputRootInputs or it starts
- // with a path in outputRootInputDirs.
- for (PathFragment input : target.getAllInputs()) {
- if (outputRootInputs.contains(input)) {
- outputRootInputsBuilder.add(input);
- } else {
- for (PathFragment outputRootInputDir : outputRootInputDirs) {
- if (input.startsWith(outputRootInputDir)) {
- outputRootInputsBuilder.add(input);
- }
- }
- }
- }
-
- if ("phony".equals(target.getRuleName())) {
- if (target.getAllOutputs().size() != 1) {
- String allOutputs =
- target.getAllOutputs().stream()
- .map(PathFragment::getPathString)
- .collect(Collectors.joining(" "));
- throw new GenericParsingException(
- String.format(
- "Ninja phony alias can only be used for single output, but found '%s'.",
- allOutputs));
- }
- phonyTargetsBuilder.put(Iterables.getOnlyElement(target.getAllOutputs()), target);
- } else {
- for (PathFragment output : target.getAllOutputs()) {
- targetsBuilder.put(output, target);
- }
- symlinkOutputsBuilder.addAll(target.getAllSymlinkOutputs());
- }
- }
- }
- }
-
- private void checkDirectoriesAttributes(
- RuleContext ruleContext, PathFragment outputRoot, PathFragment workingDirectory)
- throws InterruptedException {
- Environment env = ruleContext.getAnalysisEnvironment().getSkyframeEnv();
- ImmutableSortedSet<String> notSymlinkedDirs =
- BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER.getNotSymlinkedInExecrootDirectories(
- env);
- if (env.valuesMissing()) {
- return;
- }
-
- // We can compare strings because notSymlinkedDirs contains normalized directory names
- if (!notSymlinkedDirs.contains(outputRoot.getPathString())) {
- ruleContext.attributeError(
- "output_root",
- String.format(
- "Ninja output root directory '%s' must be declared"
- + " using global workspace function toplevel_output_directories().",
- outputRoot.getPathString()));
- }
-
- if (!workingDirectory.isEmpty() && !workingDirectory.equals(outputRoot)) {
- ruleContext.attributeError(
- "working_directory",
- String.format(
- "Ninja working directory '%s' is restricted to be either empty (or not defined),"
- + " or be the same as output root '%s'.",
- workingDirectory.getPathString(), outputRoot.getPathString()));
- }
- }
-
- /**
- * As Ninja files describe the action graph, we must establish the dependency between Ninja files
- * and the Ninja graph configured target for the SkyFrame. We are doing it by computing all
- * related FileValue SkyValues.
- */
- private static void establishDependencyOnNinjaFiles(
- Environment env, Artifact mainFile, ImmutableList<Artifact> ninjaSrcs)
- throws InterruptedException {
- Iterable<SkyKey> depKeys =
- Iterables.concat(
- ImmutableSet.of(getArtifactRootedPath(mainFile)),
- Iterables.transform(ninjaSrcs, NinjaGraph::getArtifactRootedPath));
- env.getOrderedValuesAndExceptions(depKeys);
- }
-
- private static SkyKey getArtifactRootedPath(Artifact artifact) {
- return FileValue.key(
- RootedPath.toRootedPath(artifact.getRoot().getRoot(), artifact.getRootRelativePath()));
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaGraphArtifactsHelper.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaGraphArtifactsHelper.java
deleted file mode 100644
index 8bcea43..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaGraphArtifactsHelper.java
+++ /dev/null
@@ -1,160 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.actions;
-
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.devtools.build.lib.actions.Artifact;
-import com.google.devtools.build.lib.actions.Artifact.DerivedArtifact;
-import com.google.devtools.build.lib.actions.ArtifactRoot;
-import com.google.devtools.build.lib.actions.ArtifactRoot.RootType;
-import com.google.devtools.build.lib.analysis.RuleContext;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.build.lib.vfs.Root;
-
-/**
- * Helper class to create artifacts for {@link NinjaAction} to be used from {@link NinjaGraphRule}.
- * All created output artifacts are accumulated in the NestedSetBuilder.
- *
- * <p>Input and putput paths are interpreted relative to the working directory, see
- * working_directory property in {@link NinjaGraphRule}. All output artifact are created under the
- * derived artifacts root <execroot>/<outputRoot>, see output_root property in {@link
- * NinjaGraphRule}.
- */
-class NinjaGraphArtifactsHelper {
- private final RuleContext ruleContext;
- private final PathFragment outputRootPath;
- private final PathFragment workingDirectory;
- private final ArtifactRoot derivedOutputRoot;
- private final Root sourceRoot;
-
- // Artifacts that should be symlinked directly from the source tree into the execroot.
- private final ImmutableSortedMap<PathFragment, Artifact> symlinkPathToArtifact;
-
- // Symlink output artifacts created by Ninja actions during the build.
- private final ImmutableSet<PathFragment> symlinkOutputs;
-
- /**
- * Constructor
- *
- * @param ruleContext parent NinjaGraphRule rule context
- * @param outputRootPath name of output directory for Ninja actions under execroot
- * @param workingDirectory relative path under execroot, the root for interpreting all paths in
- * Ninja file
- * @param symlinkPathToArtifact mapping of paths to artifacts for input symlinks under output_root
- * @param symlinkOutputs list of output paths for which symlink artifacts should be created, paths
- * are relative to the output_root.
- */
- NinjaGraphArtifactsHelper(
- RuleContext ruleContext,
- PathFragment outputRootPath,
- PathFragment workingDirectory,
- ImmutableSortedMap<PathFragment, Artifact> symlinkPathToArtifact,
- ImmutableSet<PathFragment> symlinkOutputs) {
- this.ruleContext = ruleContext;
- this.outputRootPath = outputRootPath;
- this.workingDirectory = workingDirectory;
- this.symlinkPathToArtifact = symlinkPathToArtifact;
- this.symlinkOutputs = symlinkOutputs;
- Path execRoot =
- Preconditions.checkNotNull(ruleContext.getConfiguration())
- .getDirectories()
- .getExecRoot(ruleContext.getWorkspaceName());
- this.derivedOutputRoot = ArtifactRoot.asDerivedRoot(execRoot, RootType.Output, outputRootPath);
- this.sourceRoot = ruleContext.getRule().getPackage().getSourceRoot().get();
- }
-
- DerivedArtifact createOutputArtifact(PathFragment pathRelativeToWorkingDirectory)
- throws GenericParsingException {
- PathFragment execPath = workingDirectory.getRelative(pathRelativeToWorkingDirectory);
-
- if (!execPath.startsWith(outputRootPath)) {
- throw new GenericParsingException(
- String.format(
- "Ninja actions are allowed to create outputs only under output_root,"
- + " path '%s' is not allowed.",
- pathRelativeToWorkingDirectory));
- }
- // If the path was declared as output symlink, create a symlink artifact.
- // symlink_outputs are always declared as relative to working directory.
- if (symlinkOutputs.contains(pathRelativeToWorkingDirectory)) {
- return ruleContext
- .getAnalysisEnvironment()
- .getSymlinkArtifact(execPath.relativeTo(outputRootPath), derivedOutputRoot);
- }
- return ruleContext.getDerivedArtifact(execPath.relativeTo(outputRootPath), derivedOutputRoot);
- }
-
- ArtifactRoot getDerivedOutputRoot() {
- return derivedOutputRoot;
- }
-
- Artifact getInputArtifact(PathFragment pathRelativeToWorkingDirectory)
- throws GenericParsingException {
- if (symlinkPathToArtifact.containsKey(pathRelativeToWorkingDirectory)) {
- return symlinkPathToArtifact.get(pathRelativeToWorkingDirectory);
- }
-
- PathFragment execPath = workingDirectory.getRelative(pathRelativeToWorkingDirectory);
- if (execPath.startsWith(outputRootPath)) {
- // This is in the output root, so it's an output artifact created from another Ninja action,
- // not a source artifact. This can be a regular DerivedArtifact or symlink SpecialArtifact,
- // depending on the set of symlinkOutputs threaded from Ninja.
- return createOutputArtifact(pathRelativeToWorkingDirectory);
- }
-
- if (!execPath.startsWith(ruleContext.getPackageDirectory())) {
- throw new GenericParsingException(
- String.format(
- "Source artifact '%s' is not under the package directory '%s' of ninja_build rule",
- execPath, ruleContext.getPackageDirectory()));
- }
-
- // Not a derived artifact. Create a corresponding source artifact. This isn't really great
- // because we have no guarantee that the artifact is not under a different package which
- // invalidates the guarantee of "bazel query" that the dependencies reported for a target are a
- // superset of all possible targets that are needed to build it, worse yet, there isn't even a
- // guarantee that there isn't a package on a different package path in between.
- //
- // For example, if the ninja_build rule is in a/BUILD and has a file a/b/c, it's possible that
- // there is a BUILD file a/b/BUILD and thus the source file a/b/c is created from the package
- // //a even though package //a/b exists (violating the above "bazel query" invariant) and it can
- // be that a/b/BUILD is on a different package path entry (is not correct because the other
- // package path entry can contain a *different* source file whose execpath is a/b/c)
- //
- // TODO(lberki): Check whether there is a package in between from another package path entry.
- // We probably can't prohibit packages in between, though. Schade.
- return ruleContext
- .getAnalysisEnvironment()
- .getSourceArtifactForNinjaBuild(
- execPath, ruleContext.getRule().getPackage().getSourceRoot().get());
- }
-
- public PathFragment getOutputRootPath() {
- return outputRootPath;
- }
-
- public PathFragment getWorkingDirectory() {
- return workingDirectory;
- }
-
- public Root getSourceRoot() {
- return sourceRoot;
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaGraphProvider.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaGraphProvider.java
deleted file mode 100644
index 49a50d1..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaGraphProvider.java
+++ /dev/null
@@ -1,77 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.actions;
-
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget;
-import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
-import com.google.devtools.build.lib.vfs.PathFragment;
-
-/**
- * Provider for passing information between {@link NinjaGraphRule} and {@link NinjaBuildRule}.
- * Represents all regular and phony {@link NinjaTarget}s from the Ninja graph.
- */
-@Immutable
-public final class NinjaGraphProvider implements TransitiveInfoProvider {
- private final PathFragment outputRoot;
- private final PathFragment workingDirectory;
- private final ImmutableSortedMap<PathFragment, NinjaTarget> targetsMap;
- private final ImmutableSortedMap<PathFragment, PhonyTarget> phonyTargetsMap;
- private final ImmutableSet<PathFragment> outputRootSymlinks;
- private final ImmutableSet<PathFragment> outputRootInputsSymlinks;
-
- public NinjaGraphProvider(
- PathFragment outputRoot,
- PathFragment workingDirectory,
- ImmutableSortedMap<PathFragment, NinjaTarget> targetsMap,
- ImmutableSortedMap<PathFragment, PhonyTarget> phonyTargetsMap,
- ImmutableSet<PathFragment> outputRootSymlinks,
- ImmutableSet<PathFragment> outputRootInputsSymlinks) {
-
- this.outputRoot = outputRoot;
- this.workingDirectory = workingDirectory;
- this.targetsMap = targetsMap;
- this.phonyTargetsMap = phonyTargetsMap;
- this.outputRootSymlinks = outputRootSymlinks;
- this.outputRootInputsSymlinks = outputRootInputsSymlinks;
- }
-
- public PathFragment getOutputRoot() {
- return outputRoot;
- }
-
- public PathFragment getWorkingDirectory() {
- return workingDirectory;
- }
-
- public ImmutableSortedMap<PathFragment, NinjaTarget> getTargetsMap() {
- return targetsMap;
- }
-
- public ImmutableSortedMap<PathFragment, PhonyTarget> getPhonyTargetsMap() {
- return phonyTargetsMap;
- }
-
- /** Output paths under output_root, that should be treated as symlink artifacts. */
- public ImmutableSet<PathFragment> getOutputRootSymlinks() {
- return outputRootSymlinks;
- }
-
- public ImmutableSet<PathFragment> getOutputRootInputsSymlinks() {
- return outputRootInputsSymlinks;
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaGraphRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaGraphRule.java
deleted file mode 100644
index 7e4e3f1..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaGraphRule.java
+++ /dev/null
@@ -1,108 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.actions;
-
-import static com.google.devtools.build.lib.packages.Attribute.attr;
-import static com.google.devtools.build.lib.packages.BuildType.LABEL;
-import static com.google.devtools.build.lib.packages.BuildType.LABEL_LIST;
-import static com.google.devtools.build.lib.packages.Type.STRING;
-import static com.google.devtools.build.lib.packages.Type.STRING_LIST;
-
-import com.google.common.collect.ImmutableList;
-import com.google.devtools.build.lib.analysis.BaseRuleClasses;
-import com.google.devtools.build.lib.analysis.RuleDefinition;
-import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
-import com.google.devtools.build.lib.packages.RuleClass;
-import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
-import com.google.devtools.build.lib.util.FileTypeSet;
-import net.starlark.java.eval.Sequence;
-import net.starlark.java.eval.StarlarkThread;
-
-/**
- * The rule that parses the Ninja graph and symlinks inputs into output_root.
- *
- * <p>The rule exposes {@link NinjaGraphProvider} with maps of both non-phony and phony {@link
- * com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget} for {@link NinjaBuildRule} to
- * use for action creation.
- *
- * <p>The rules establishes Skyframe dependency on input Ninja files, as each time they change, the
- * action graph changes.
- *
- * <p>Important aspect is relation to non-symlinked-under-execroot-directories: {@link
- * com.google.devtools.build.lib.starlarkbuildapi.WorkspaceGlobalsApi#dontSymlinkDirectoriesInExecroot(Sequence,
- * StarlarkThread)} All the outputs of Ninja actions are expected to be under the directory,
- * specified in output_root of this rule. All the input files under output_root should be listed in
- * output_root_inputs attribute, this rule will create the SymlinkAction actions to symlink listed
- * files under <execroot>/<output_root>.
- */
-public class NinjaGraphRule implements RuleDefinition {
- @Override
- public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment environment) {
- return builder
- .add(
- attr("ninja_srcs", LABEL_LIST)
- .allowedFileTypes(FileTypeSet.ANY_FILE)
- .setDoc("All included or subninja Ninja files describing the action graph."))
- .add(
- attr("main", LABEL)
- .allowedFileTypes(FileTypeSet.ANY_FILE)
- .mandatory()
- .setDoc("Main Ninja file."))
- .add(
- attr("output_root", STRING)
- .mandatory()
- .setDoc(
- "<p>Directory under workspace, where all the intermediate and output artifacts"
- + " will be created.</p><p>Must not be symlinked to the execroot. For"
- + " that, toplevel_output_directories function should be used in"
- + " WORKSPACE file.</p>"))
- .add(
- attr("output_root_inputs", STRING_LIST)
- .value(ImmutableList.of())
- .setDoc(
- "<p>Paths under output_root, that are used as inputs to the Ninja"
- + " file.</p><p>For each path, an action to symlink under"
- + " <execroot>/<output_root> will be created by this rule."
- + " <execroot>/<output_root> will be a separate directory, not a"
- + " symlink.</p>"))
- .add(
- attr("output_root_input_dirs", STRING_LIST)
- .value(ImmutableList.of())
- .setDoc(
- "<p>Directory paths under output_root that contain files (and subdirectories"
- + " of files) to be used as inputs to the Ninja file.</p><p>For each child"
- + " path of an input directory which is referenced in the ninja file, an"
- + " action to symlink under <execroot>/<output_root> will be created by"
- + " this rule.</p>"))
- .add(
- attr("working_directory", STRING)
- .value("")
- .setDoc(
- "Directory under workspace's exec root to be the root for relative paths and "
- + "working directory for all Ninja actions. "
- + "Must be empty or set to the value or output_root."))
- .build();
- }
-
- @Override
- public Metadata getMetadata() {
- return RuleDefinition.Metadata.builder()
- .name("ninja_graph")
- .type(RuleClassType.NORMAL)
- .ancestors(BaseRuleClasses.NativeBuildRule.class)
- .factoryClass(NinjaGraph.class)
- .build();
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaPhonyTargetsUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaPhonyTargetsUtil.java
deleted file mode 100644
index 9719dcd..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaPhonyTargetsUtil.java
+++ /dev/null
@@ -1,153 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.actions;
-
-import static com.google.common.base.Preconditions.checkState;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget;
-import com.google.devtools.build.lib.util.Pair;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.util.ArrayDeque;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-
-/**
- * An utility class for gathering non-phony input dependencies of phony {@link NinjaTarget} objects
- * from some Ninja file.
- *
- * <p>Cycles are detected and {@link GenericParsingException} is thrown in that case.
- */
-public class NinjaPhonyTargetsUtil {
-
- private NinjaPhonyTargetsUtil() {}
-
- @VisibleForTesting
- public static ImmutableSortedMap<PathFragment, PhonyTarget> getPhonyPathsMap(
- ImmutableSortedMap<PathFragment, NinjaTarget> phonyTargets) throws GenericParsingException {
- // There is always a DAG (or forest) of phony targets (as item can be included into several
- // phony targets).
- // This gives us the idea that we can compute any subgraph in the DAG independently, and
- // later include that subgraph into the parent DAG.
- // In the list 'topoOrderedTargets' we will put all the NinjaTargets from the phonyTargets,
- // in the following order: all nodes from a node's subtree are preceding it in a list.
- // This way we can later visit the list "topoOrderedTargets", computing the input files
- // for targets, and for each target K the results for all the phony targets from a subgraph of K
- // will be already computed.
- // The sorting is linear, as we are only checking each input of each node once (we use already).
- List<NinjaTarget> topoOrderedTargets = Lists.newArrayListWithCapacity(phonyTargets.size());
- Set<NinjaTarget> alreadyVisited = Sets.newHashSet();
- for (Map.Entry<PathFragment, NinjaTarget> entry : phonyTargets.entrySet()) {
- NinjaTarget target = entry.getValue();
- topoOrderedTargets.addAll(topoOrderSubGraph(phonyTargets, alreadyVisited, target));
- }
-
- checkState(topoOrderedTargets.size() == phonyTargets.size());
-
- SortedMap<PathFragment, PhonyTarget> result = Maps.newTreeMap();
- for (NinjaTarget target : topoOrderedTargets) {
- PathFragment onlyOutput = Iterables.getOnlyElement(target.getAllOutputs());
- ImmutableList.Builder<PathFragment> phonyNames = ImmutableList.builder();
- ImmutableList.Builder<PathFragment> directInputs = ImmutableList.builder();
- Collection<PathFragment> allInputs = target.getAllInputs();
- boolean isAlwaysDirty = allInputs.isEmpty();
- for (PathFragment input : allInputs) {
- NinjaTarget phonyInput = phonyTargets.get(input);
- if (phonyInput != null) {
- // The input is the other phony target.
- // Phony target must have only one output (alias); it is checked during parsing.
- PathFragment phonyName = Iterables.getOnlyElement(phonyInput.getAllOutputs());
- PhonyTarget alreadyComputed = result.get(phonyName);
- Preconditions.checkNotNull(alreadyComputed);
- isAlwaysDirty |= alreadyComputed.isAlwaysDirty();
- phonyNames.add(Iterables.getOnlyElement(phonyInput.getAllOutputs()));
- } else {
- // The input is an explicit input file.
- directInputs.add(input);
- }
- }
- result.put(
- onlyOutput, new PhonyTarget(phonyNames.build(), directInputs.build(), isAlwaysDirty));
- }
-
- return ImmutableSortedMap.copyOf(result);
- }
-
- /**
- * For the given phony NinjaTarget, return a list of all phony NinjaTargets, composing its subtree
- * (direct and transitive inputs). The list is ordered from leaves to their dependents; for any
- * node all its direct and transitive inputs are preceding it in the list.
- *
- * <p>Function does DFS starting from the NinjaTarget, with two phases: in initial processing: 1)
- * if the target was already computed, nothing happens 2) the target is checked for cycle and
- * marked in cycleProtection set, its phony inputs are queued (put in the beginning of the queue)
- * for initial processing 3) the target is queued after its inputs for post-processing in
- * post-processing, the target is recorded into resulting list; all its inputs should have been
- * already written to that list on the previous steps
- */
- private static List<NinjaTarget> topoOrderSubGraph(
- ImmutableSortedMap<PathFragment, NinjaTarget> phonyTargets,
- Set<NinjaTarget> alreadyVisited,
- NinjaTarget target)
- throws GenericParsingException {
- Set<NinjaTarget> cycleProtection = Sets.newHashSet();
- List<NinjaTarget> fragment = Lists.newArrayList();
- ArrayDeque<Pair<NinjaTarget, Boolean>> queue = new ArrayDeque<>();
- queue.add(Pair.of(target, true));
- while (!queue.isEmpty()) {
- Pair<NinjaTarget, Boolean> pair = queue.remove();
- NinjaTarget currentTarget = pair.getFirst();
- if (pair.getSecond()) {
- // Initial processing: checking all the phony inputs of the current target.
- if (alreadyVisited.contains(currentTarget)) {
- continue;
- }
- if (!cycleProtection.add(currentTarget)) {
- throw new GenericParsingException(
- String.format(
- "Detected a dependency cycle involving the phony target '%s'",
- Iterables.getOnlyElement(currentTarget.getAllOutputs())));
- }
- // Adding <phony-inputs-of-current-target> for initial processing in front of
- // <current-target>
- // for post-processing into the queue.
- queue.addFirst(Pair.of(currentTarget, false));
- for (PathFragment input : currentTarget.getAllInputs()) {
- NinjaTarget phonyInput = phonyTargets.get(input);
- if (phonyInput != null) {
- queue.addFirst(Pair.of(phonyInput, true));
- }
- }
- } else {
- // Post processing: all inputs should have been processed and added to fragment.
- cycleProtection.remove(currentTarget);
- alreadyVisited.add(currentTarget);
- fragment.add(currentTarget);
- }
- }
- return fragment;
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaRulesModule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaRulesModule.java
deleted file mode 100644
index 4e48b20..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/NinjaRulesModule.java
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.actions;
-
-import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
-import com.google.devtools.build.lib.runtime.BlazeModule;
-
-/** Module for Ninja execution. */
-public class NinjaRulesModule extends BlazeModule {
- @Override
- public void initializeRuleClasses(ConfiguredRuleClassProvider.Builder builder) {
- builder.addRuleDefinition(new NinjaGraphRule());
- builder.addRuleDefinition(new NinjaBuildRule());
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/PhonyTarget.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/PhonyTarget.java
deleted file mode 100644
index 0c61c5b..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/PhonyTarget.java
+++ /dev/null
@@ -1,93 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.actions;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Sets;
-import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.util.ArrayDeque;
-import java.util.Set;
-import java.util.function.Consumer;
-
-/**
- * Helper class to represent "evaluated" Ninja phony target.
- *
- * <p>"phony" is a special rule in Ninja that behaves somewhat like an alias for targets.
- *
- * <p>e.g. `build foo: phony bar/qux.txt` makes `foo` a phony target for the `bar/qux.txt` non-phony
- * target, so `ninja foo` will run the commands that build `bar/qux.txt`. For more information, see
- * the docs for Ninja phony targets:
- *
- * <p>https://ninja-build.org/manual.html#_the_literal_phony_literal_rule
- *
- * <p>A PhonyTarget contains the List with direct non-phony inputs to the phony target
- * (PathFragments), the list with direct phony inputs; and it contains the flag whether this phony
- * target is always dirty, i.e. must be rebuild each time.
- *
- * <p>Always-dirty phony targets are those which do not have any inputs: "build alias: phony". All
- * non-phony direct dependants of those actions automatically also always-dirty (but not the
- * transitive dependants: they should check whether their computed inputs have changed). As phony
- * targets are not performing any actions, <b>all phony transitive dependants of always-dirty phony
- * targets are themselves always-dirty.</b> That is why we can compute the always-dirty flag for the
- * phony targets, and use it for marking their direct non-phony dependants as actions to be executed
- * unconditionally.
- */
-@Immutable
-public final class PhonyTarget {
- private final ImmutableList<PathFragment> phonyNames;
- private final ImmutableList<PathFragment> directExplicitInputs;
- private final boolean isAlwaysDirty;
-
- public PhonyTarget(
- ImmutableList<PathFragment> phonyNames,
- ImmutableList<PathFragment> directExplicitInputs,
- boolean isAlwaysDirty) {
- this.phonyNames = phonyNames;
- this.directExplicitInputs = directExplicitInputs;
- this.isAlwaysDirty = isAlwaysDirty;
- }
-
- public ImmutableList<PathFragment> getPhonyNames() {
- return phonyNames;
- }
-
- public ImmutableList<PathFragment> getDirectExplicitInputs() {
- return directExplicitInputs;
- }
-
- public boolean isAlwaysDirty() {
- return isAlwaysDirty;
- }
-
- public void visitExplicitInputs(
- ImmutableSortedMap<PathFragment, PhonyTarget> phonyTargetsMap,
- Consumer<ImmutableList<PathFragment>> consumer) {
- consumer.accept(directExplicitInputs);
-
- ArrayDeque<PathFragment> queue = new ArrayDeque<>(phonyNames);
- Set<PathFragment> visited = Sets.newHashSet();
- while (!queue.isEmpty()) {
- PathFragment fragment = queue.remove();
- if (visited.add(fragment)) {
- PhonyTarget phonyTarget = Preconditions.checkNotNull(phonyTargetsMap.get(fragment));
- consumer.accept(phonyTarget.getDirectExplicitInputs());
- queue.addAll(phonyTarget.getPhonyNames());
- }
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/PhonyTargetArtifacts.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/PhonyTargetArtifacts.java
deleted file mode 100644
index 96c343e..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions/PhonyTargetArtifacts.java
+++ /dev/null
@@ -1,69 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.actions;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Maps;
-import com.google.devtools.build.lib.actions.Artifact;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.collect.nestedset.NestedSet;
-import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.util.Map;
-import javax.annotation.concurrent.ThreadSafe;
-
-/**
- * Helper class for caching computation of transitive inclusion of non-phony targets into phony
- * ones. We cannot compute all artifacts for all phony targets because some of them may not be
- * created by a subgraph of required actions.
- */
-@ThreadSafe
-public class PhonyTargetArtifacts {
- private final Map<PathFragment, NestedSet<Artifact>> cache;
- private final ImmutableSortedMap<PathFragment, PhonyTarget> phonyTargetsMap;
- private final NinjaGraphArtifactsHelper artifactsHelper;
-
- public PhonyTargetArtifacts(
- ImmutableSortedMap<PathFragment, PhonyTarget> phonyTargetsMap,
- NinjaGraphArtifactsHelper artifactsHelper) {
- this.phonyTargetsMap = phonyTargetsMap;
- this.artifactsHelper = artifactsHelper;
- cache = Maps.newHashMap();
- }
-
- NestedSet<Artifact> getPhonyTargetArtifacts(PathFragment name) throws GenericParsingException {
- NestedSet<Artifact> existing = cache.get(name);
- if (existing != null) {
- return existing;
- }
- PhonyTarget phonyTarget = phonyTargetsMap.get(name);
- Preconditions.checkNotNull(phonyTarget);
- NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
- for (PathFragment input : phonyTarget.getDirectExplicitInputs()) {
- builder.add(artifactsHelper.getInputArtifact(input));
- }
- for (PathFragment phonyName : phonyTarget.getPhonyNames()) {
- // We already checked for cycles during loading.
- NestedSet<Artifact> nestedSet = getPhonyTargetArtifacts(phonyName);
- builder.addTransitive(nestedSet);
- }
- NestedSet<Artifact> value = builder.build();
- // We do not hold the lock during the computation, so deadlocks are not possible,
- // however duplicate computations are possible.
- cache.put(name, value);
- return value;
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/BUILD
deleted file mode 100644
index e22e0e6..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/BUILD
+++ /dev/null
@@ -1,17 +0,0 @@
-load("@rules_java//java:defs.bzl", "java_library")
-
-package(
- default_visibility = ["//src:__subpackages__"],
-)
-
-filegroup(
- name = "srcs",
- srcs = glob(["*"]),
- visibility = ["//src:__subpackages__"],
-)
-
-java_library(
- name = "file",
- srcs = glob(["*.java"]),
- deps = ["//third_party:guava"],
-)
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/CollectingListFuture.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/CollectingListFuture.java
deleted file mode 100644
index 188f672..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/CollectingListFuture.java
+++ /dev/null
@@ -1,60 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.file;
-
-import com.google.common.base.Throwables;
-import com.google.common.collect.Lists;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-
-/**
- * Helper class for accumulating the scheduled {@link ListenableFuture} sub-tasks and gathering
- * their results.
- *
- * @param <T> the type that ListenableFuture returns
- * @param <E> the type of exception, that the computation task can throw
- */
-public class CollectingListFuture<T, E extends Exception> {
- private final List<ListenableFuture<T>> futures;
- private final Class<E> exceptionClazz;
-
- /** @param exceptionClazz the class of exception, that the computation task can throw */
- public CollectingListFuture(Class<E> exceptionClazz) {
- this.exceptionClazz = exceptionClazz;
- futures = Lists.newArrayList();
- }
-
- /** Adds future to the list */
- public void add(ListenableFuture<T> future) {
- futures.add(future);
- }
-
- /** Returns the list of combined results of all futures, registered with {@link #add}. */
- public List<T> getResult() throws E, InterruptedException {
- try {
- return Futures.allAsList(futures).get();
- } catch (ExecutionException e) {
- Throwable causeOrSelf = e.getCause();
- if (causeOrSelf == null) {
- causeOrSelf = e;
- }
- Throwables.propagateIfPossible(causeOrSelf, exceptionClazz);
- throw new IllegalStateException(e);
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/DeclarationAssembler.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/DeclarationAssembler.java
deleted file mode 100644
index 18943b3..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/DeclarationAssembler.java
+++ /dev/null
@@ -1,125 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.file;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-
-/**
- * A {@link FileFragmentSplitter} callback interface implementation, that assembles fragments of
- * declarations (that may occur on the edges of byte buffer fragments) together and passes all
- * declarations to delegate {@link DeclarationConsumer}, which does further processing / parsing.
- */
-public class DeclarationAssembler {
- private final DeclarationConsumer declarationConsumer;
-
- /** @param declarationConsumer delegate declaration consumer for actual processing / parsing */
- public DeclarationAssembler(DeclarationConsumer declarationConsumer) {
- this.declarationConsumer = declarationConsumer;
- }
-
- /**
- * Should be called after all work for processing of individual buffer fragments is complete.
- *
- * @param fragments list of {@link FileFragment} - pieces on the bounds of sub-fragments.
- * @throws GenericParsingException thrown by delegate {@link #declarationConsumer}
- */
- public void wrapUp(List<FileFragment> fragments) throws GenericParsingException, IOException {
- fragments.sort(Comparator.comparingLong(FileFragment::getFragmentOffset));
-
- List<FileFragment> list = Lists.newArrayList();
- long previous = -1;
- for (FileFragment edge : fragments) {
- long start = edge.getFragmentOffset();
- FileFragment fragment = edge;
- if (previous >= 0 && previous != start) {
- sendMerged(list);
- list.clear();
- }
- list.add(edge);
- previous = start + fragment.length();
- }
- if (!list.isEmpty()) {
- sendMerged(list);
- }
- }
-
- private void sendMerged(List<FileFragment> list) throws GenericParsingException, IOException {
- Preconditions.checkArgument(!list.isEmpty());
- FileFragment first = list.get(0);
- if (list.size() == 1) {
- declarationConsumer.declaration(first);
- return;
- }
-
- // 1. We merge all the passed fragments into one fragment.
- // 2. We check 6 bytes at the connection of two fragments, 3 bytes in each part:
- // separator can consist of 4 bytes (<escape>/r/n<indent>),
- // so in case only a part of the separator is in one of the fragments,
- // we get 3 bytes in one part and one byte in the other.
- // 3. We record the ranges of at most 6 bytes at the connections of the fragments into
- // interestingRanges.
- // 4. Later we will check only interestingRanges for separators, and create corresponding
- // fragments; the underlying common ByteBuffer will be reused, so we are not performing
- // extensive copying.
- List<FileFragment> fragments = new ArrayList<>();
- List<Range<Integer>> interestingRanges = Lists.newArrayList();
- int fragmentShift = 0;
- for (FileFragment fragment : list) {
- fragments.add(fragment);
- if (fragmentShift > 0) {
- // We are only looking for the separators between fragments.
- int start = Math.max(0, fragmentShift - 3);
- int end = fragmentShift + Math.min(4, fragment.length());
- Range<Integer> candidate = Range.openClosed(start, end);
- if (!interestingRanges.isEmpty()
- && Iterables.getLast(interestingRanges).isConnected(candidate)) {
- // If the last fragment was tiny, it can be that the ranges before the last fragment and
- // the one before this fragment intersect. In that case, merge them.
- interestingRanges.set(
- interestingRanges.size() - 1, Iterables.getLast(interestingRanges).span(candidate));
- } else {
- interestingRanges.add(candidate);
- }
- }
- fragmentShift += fragment.length();
- }
-
- FileFragment merged = FileFragment.merge(fragments);
-
- int previousEnd = 0;
- for (Range<Integer> range : interestingRanges) {
- int idx =
- NinjaSeparatorFinder.findNextSeparator(
- merged, range.lowerEndpoint(), range.upperEndpoint());
- if (idx >= 0) {
- // There should always be a previous fragment, as we are checking non-intersecting ranges,
- // starting from the connection point between first and second fragments.
- Preconditions.checkState(idx > previousEnd);
- declarationConsumer.declaration(merged.subFragment(previousEnd, idx + 1));
- previousEnd = idx + 1;
- }
- }
-
- declarationConsumer.declaration(merged.subFragment(previousEnd, merged.length()));
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/DeclarationConsumer.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/DeclarationConsumer.java
deleted file mode 100644
index 48b7885..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/DeclarationConsumer.java
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.file;
-
-import java.io.IOException;
-
-/** Generic interface to accept declarations from {@link ParallelFileProcessing} */
-public interface DeclarationConsumer {
-
- /**
- * Accepts a declaration for further processing / parsing.
- *
- * @param fragment a fragment of {@link java.nio.ByteBuffer} in the form of {@link FileFragment},
- * starting at offset in the underlying file.
- * @throws GenericParsingException if declaration processing discovered the wrong syntax
- */
- void declaration(FileFragment fragment) throws GenericParsingException, IOException;
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/FileFragment.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/FileFragment.java
deleted file mode 100644
index 58a7c38..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/FileFragment.java
+++ /dev/null
@@ -1,204 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.file;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-/** Represents the fragment of immutable {@link ByteBuffer} for parallel processing. */
-public class FileFragment {
- /** The backing {@link ByteBuffer}. May be shared with other {@link FileFragment} instances. */
- private final ByteBuffer buffer;
-
- /** The offset in the file the backing {@link ByteBuffer} starts at. */
- private final long fileOffset;
-
- /**
- * The start of this fragment in the backing {@link ByteBuffer}.
- *
- * <p>The file offset of the fragment is the sum of this value and the offset of the backing
- * buffer ({@code fileOffset}).
- */
- private final int startIncl;
-
- /** The end of this fragment in the backing {@link ByteBuffer}. */
- private final int endExcl;
-
- public FileFragment(ByteBuffer buffer, long fileOffset, int startIncl, int endExcl) {
- if (endExcl < startIncl) {
- throw new IndexOutOfBoundsException();
- }
- this.buffer = buffer;
- this.fileOffset = fileOffset;
- this.startIncl = startIncl;
- this.endExcl = endExcl;
- }
-
- /**
- * Returns the start of this fragment in the backing {@link ByteBuffer}, that is, relative to the
- * {@code fileOffset} in the file.
- */
- @VisibleForTesting
- public int getStartIncl() {
- return startIncl;
- }
-
- /** The byte offset in the file the backing {@link ByteBuffer} starts at. */
- public long getFileOffset() {
- return fileOffset;
- }
-
- /** The offset in the file this fragment starts at. */
- public long getFragmentOffset() {
- return fileOffset + startIncl;
- }
-
- /** Returns the length of fragment. */
- public int length() {
- return endExcl - startIncl;
- }
-
- /**
- * Creates a sub-fragment of the current fragment.
- *
- * @param from start index, inclusive
- * @param to end index, exclusive, or the size of the buffer, if the last symbol is included
- */
- public FileFragment subFragment(int from, int to) {
- checkSubBounds(from, to);
- return new FileFragment(buffer, fileOffset, startIncl + from, startIncl + to);
- }
-
- public byte byteAt(int index) {
- if (index < 0 || index >= length()) {
- throw new IndexOutOfBoundsException(
- String.format("Index out of bounds: %d (%d, %d).", index, 0, length()));
- }
- return buffer.get(startIncl + index);
- }
-
- private void checkSubBounds(int from, int to) {
- if (from < 0) {
- throw new IndexOutOfBoundsException(String.format("Index out of bounds: %d.", from));
- }
- if (to > length()) {
- throw new IndexOutOfBoundsException(
- String.format("Index out of bounds: %d (%d, %d).", to, 0, length()));
- }
- if (from > to) {
- throw new IndexOutOfBoundsException(
- String.format("Start index is greater than end index: %d, %d.", from, to));
- }
- }
-
- /**
- * Helper method for forming error messages with text fragment around a place with a problem.
- *
- * @param index position of the problematic symbol.
- * @return a fragment if text around the problematic place, that can be retrieved from this buffer
- */
- public String getFragmentAround(int index) {
- if (index < 0 || index >= length()) {
- throw new IndexOutOfBoundsException(
- String.format("Index out of bounds: %d (%d, %d).", index, 0, length()));
- }
- byte[] bytes = getBytes(Math.max(0, index - 200), Math.min(length(), index + 200));
- return new String(bytes, StandardCharsets.ISO_8859_1);
- }
-
- public byte[] getBytes(int from, int to) {
- checkSubBounds(from, to);
- int length = to - from;
- byte[] bytes = new byte[length];
- ByteBuffer copy = buffer.duplicate();
- copy.position(startIncl + from);
- copy.get(bytes, 0, length);
- return bytes;
- }
-
- private void getBytes(byte[] dst, int offset) {
- if (dst.length - offset < length()) {
- throw new IndexOutOfBoundsException(
- String.format(
- "Not enough space to copy: %d available, %d needed.",
- (dst.length - offset), length()));
- }
- ByteBuffer copy = buffer.duplicate();
- copy.position(startIncl);
- copy.get(dst, offset, (endExcl - startIncl));
- }
-
- @Override
- public String toString() {
- byte[] bytes = new byte[length()];
- getBytes(bytes, 0);
- return new String(bytes, StandardCharsets.ISO_8859_1);
- }
-
- /** Decode the underlying bytes using provided charset. */
- public String toString(Charset charset) {
- ByteBuffer duplicate = buffer.duplicate();
- duplicate.limit(endExcl);
- duplicate.position(startIncl);
- return charset.decode(duplicate).toString();
- }
-
- /**
- * Merges multiple file fragments into a single one.
- *
- * <p>If needed, also creates a new backing {@link ByteBuffer} (if the fragments on the input were
- * not contiguous in a single buffer)
- *
- * @param list non-empty list of file fragments
- * @return the merged file fragment
- */
- @SuppressWarnings("ReferenceEquality")
- public static FileFragment merge(List<FileFragment> list) {
- Preconditions.checkState(!list.isEmpty());
- if (list.size() == 1) {
- return list.get(0);
- }
- ByteBuffer first = list.get(0).buffer;
- long firstOffset = list.get(0).fileOffset;
-
- // We compare contained fragments to be exactly same objects here, i.e. changing to use
- // of equals() is not needed. (Warning suppressed.)
- List<FileFragment> tail = list.subList(1, list.size());
- if (tail.stream().allMatch(el -> el.buffer == first)) {
- int previousEnd = list.get(0).endExcl;
- for (FileFragment fragment : tail) {
- Preconditions.checkState(fragment.startIncl == previousEnd);
- previousEnd = fragment.endExcl;
- }
- return new FileFragment(
- first, firstOffset, list.get(0).startIncl, Iterables.getLast(list).endExcl);
- }
- int len = list.stream().mapToInt(FileFragment::length).sum();
- byte[] bytes = new byte[len];
- int position = 0;
- for (FileFragment current : list) {
- current.getBytes(bytes, position);
- position += current.length();
- }
- return new FileFragment(
- ByteBuffer.wrap(bytes), firstOffset + list.get(0).getStartIncl(), 0, len);
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/FileFragmentSplitter.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/FileFragmentSplitter.java
deleted file mode 100644
index 6db928e..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/FileFragmentSplitter.java
+++ /dev/null
@@ -1,71 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.file;
-
-import com.google.common.collect.Lists;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.concurrent.Callable;
-
-/**
- * Task for splitting the contents of the {@link FileFragment} (with underlying {@link ByteBuffer}).
- * Intended to be called in parallel for the fragments of the {@link ByteBuffer} for lexing the
- * contents into independent logical declarations.
- *
- * <p>{@link ParallelFileProcessing}
- */
-public class FileFragmentSplitter implements Callable<List<FileFragment>> {
- private final FileFragment fileFragment;
- private final DeclarationConsumer consumer;
-
- /**
- * @param fileFragment {@link FileFragment}, fragment of which should be splitted
- * @param consumer declaration consumer
- */
- public FileFragmentSplitter(FileFragment fileFragment, DeclarationConsumer consumer) {
- this.fileFragment = fileFragment;
- this.consumer = consumer;
- }
-
- /**
- * Returns the list of {@link FileFragment} - fragments on the bounds of the current fragment,
- * which should be potentially merged with neighbor fragments.
- *
- * <p>Combined list of {@link FileFragment} is passed to {@link DeclarationAssembler} for merging.
- */
- @Override
- public List<FileFragment> call() throws Exception {
- List<FileFragment> fragments = Lists.newArrayList();
- int start = 0;
- while (true) {
- int end = NinjaSeparatorFinder.findNextSeparator(fileFragment, start, -1);
- if (end < 0) {
- break;
- }
- FileFragment fragment = fileFragment.subFragment(start, end + 1);
- if (start > 0) {
- consumer.declaration(fragment);
- } else {
- fragments.add(fragment);
- }
- start = end + 1;
- }
- // There is always at least one byte at the bounds of the fragment.
- FileFragment lastFragment = fileFragment.subFragment(start, fileFragment.length());
- fragments.add(lastFragment);
- return fragments;
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/GenericParsingException.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/GenericParsingException.java
deleted file mode 100644
index af854f4..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/GenericParsingException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.file;
-
-/**
- * Exception to be used with parallel file fragments tokenizing/parsing in {@link
- * DeclarationConsumer}
- */
-public class GenericParsingException extends Exception {
- public GenericParsingException(String message) {
- super(message);
- }
-
- public GenericParsingException(String message, Throwable cause) {
- super(message, cause);
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/IncorrectSeparatorException.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/IncorrectSeparatorException.java
deleted file mode 100644
index 7c712d1..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/IncorrectSeparatorException.java
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.file;
-
-/** Thrown by {@link FileFragmentSplitter} when incorrect file separators are used ('\r'). */
-public class IncorrectSeparatorException extends GenericParsingException {
- public IncorrectSeparatorException(String message) {
- super(message);
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/NinjaSeparatorFinder.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/NinjaSeparatorFinder.java
deleted file mode 100644
index af88b14..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/NinjaSeparatorFinder.java
+++ /dev/null
@@ -1,75 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.file;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Finds the border between two different Ninja declarations.
- *
- * <p>The Ninja declaration consists of several text lines; if the line is a part of the previous
- * declaration, it starts with some amount of spaces or tabs. If the line is the beginning of the
- * new declaration, it starts with non-space symbol. Dollar symbol '$' escapes the newline, i.e.
- * "$\nsomething" does not contain a separator.
- *
- * <p>We support '\r\n' separators in Ninja files and throw {@link IncorrectSeparatorException} in
- * case an incorrect separator '\r' is used.
- */
-public class NinjaSeparatorFinder {
- private static final byte DOLLAR_BYTE = '$';
- private static final byte LINEFEED_BYTE = '\r';
- private static final byte NEWLINE_BYTE = '\n';
- private static final byte SPACE_BYTE = ' ';
- private static final byte TAB_BYTE = '\t';
-
- private NinjaSeparatorFinder() {}
-
- public static int findNextSeparator(FileFragment fragment, int startingFrom, int untilExcluded)
- throws IncorrectSeparatorException {
- Preconditions.checkState(startingFrom < fragment.length());
- Preconditions.checkState(untilExcluded < 0 || untilExcluded <= fragment.length());
-
- boolean escaped = DOLLAR_BYTE == fragment.byteAt(startingFrom);
- int endExcl = untilExcluded > 0 ? untilExcluded : fragment.length();
- for (int i = startingFrom + 1; i < endExcl - 1; i++) {
- byte current = fragment.byteAt(i);
- byte next = fragment.byteAt(i + 1);
- byte afterNextOrSpace = i < (endExcl - 2) ? fragment.byteAt(i + 2) : SPACE_BYTE;
- if (LINEFEED_BYTE == current && NEWLINE_BYTE != next) {
- throw new IncorrectSeparatorException(
- "Wrong newline separators: \\r should be followed by \\n.");
- }
- if (!escaped
- && SPACE_BYTE != afterNextOrSpace
- && TAB_BYTE != afterNextOrSpace
- && LINEFEED_BYTE == current) {
- // To do not introduce the length of the separator, let us point to the last symbol of it.
- return i + 1;
- }
- if (!escaped && SPACE_BYTE != next && TAB_BYTE != next && NEWLINE_BYTE == current) {
- return i;
- }
- if (escaped && LINEFEED_BYTE == current) {
- // Jump over the whole escaped linefeed + newline.
- ++i;
- escaped = false;
- } else {
- escaped = DOLLAR_BYTE == current;
- }
- }
- return -1;
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/ParallelFileProcessing.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/ParallelFileProcessing.java
deleted file mode 100644
index f18d0ab..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file/ParallelFileProcessing.java
+++ /dev/null
@@ -1,210 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.file;
-
-import com.google.common.util.concurrent.ListeningExecutorService;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.ReadableByteChannel;
-import java.util.List;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-
-/**
- * Parallel file processing implementation. See comment to {@link #processFile(ReadableByteChannel,
- * BlockParameters, Supplier, ListeningExecutorService)}.
- */
-public class ParallelFileProcessing {
- private final ReadableByteChannel channel;
- private final BlockParameters parameters;
- private final Supplier<DeclarationConsumer> tokenConsumerFactory;
- private final ListeningExecutorService executorService;
-
- private ParallelFileProcessing(
- ReadableByteChannel channel,
- BlockParameters parameters,
- Supplier<DeclarationConsumer> tokenConsumerFactory,
- ListeningExecutorService executorService) {
- this.channel = channel;
- this.parameters = parameters;
- this.tokenConsumerFactory = tokenConsumerFactory;
- this.executorService = executorService;
- }
-
- /**
- * Processes file in parallel: {@link java.nio.channels.FileChannel} is used to read contents into
- * a sequence of buffers. Each buffer is split into chunks, which are tokenized in parallel by
- * {@link FileFragmentSplitter}, using the <code>predicate</code>. Fragments of tokens (on the
- * bounds of buffer fragments) are assembled by {@link DeclarationAssembler}. The resulting tokens
- * (each can be further parsed independently) are passed to {@link DeclarationConsumer}.
- *
- * <p>The main ideas behind this implementation are:
- *
- * <p>1) using the NIO with direct buffer allocation for reading from file, (a quote from
- * ByteBuffer's javadoc: "Given a direct byte buffer, the Java virtual machine will make a best
- * effort to perform native I/O operations directly upon it. That is, it will attempt to avoid
- * copying the buffer's content to (or from) an intermediate buffer before (or after) each
- * invocation of one of the underlying operating system's native I/O operations.")
- *
- * <p>2) utilizing the possibilities for parallel processing, since splitting into tokens and
- * parsing them can be done in high degree independently.
- *
- * <p>3) not creating additional copies of character buffers for keeping tokens and only then
- * specific objects.
- *
- * <p>4) for absorbing the results, it is possible to create a consumer for each tokenizer, and
- * escape synchronization, summarizing the results after all parallel work is done, of use just
- * one shared consumer with synchronized data structures.
- *
- * <p>Please see a comment about performance test results: {@link
- * com.google.devtools.build.lib.bazel.rules.ninja.ParallelFileProcessingTest}.
- *
- * @param channel open {@link ReadableByteChannel} to file to process. The lifetime of the channel
- * is outside the scope of this method. Channel should not be closed by this method.
- * @param parameters {@link BlockParameters} with sizes of read and tokenize blocks
- * @param tokenConsumerFactory factory of {@link DeclarationConsumer} for further processing /
- * parsing
- * @param executorService executorService to use for parallel tokenization tasks
- * @param predicate predicate for separating tokens
- * @throws GenericParsingException thrown by further processing in <code>tokenConsumer</code>
- * @throws IOException thrown by file reading
- */
- public static void processFile(
- ReadableByteChannel channel,
- BlockParameters parameters,
- Supplier<DeclarationConsumer> tokenConsumerFactory,
- ListeningExecutorService executorService)
- throws GenericParsingException, IOException, InterruptedException {
- new ParallelFileProcessing(channel, parameters, tokenConsumerFactory, executorService)
- .processFileImpl();
- }
-
- private void processFileImpl() throws InterruptedException, IOException, GenericParsingException {
- if (parameters.readBlockSize == 0) {
- // Return immediately, if the file is empty.
- return;
- }
- DeclarationAssembler assembler = new DeclarationAssembler(tokenConsumerFactory.get());
-
- CollectingListFuture<List<FileFragment>, GenericParsingException> future =
- new CollectingListFuture<>(GenericParsingException.class);
- List<List<FileFragment>> listOfLists;
- long offset = 0;
- boolean keepReading = true;
- while (keepReading) {
- ByteBuffer bb = ByteBuffer.allocateDirect(parameters.getReadBlockSize());
- keepReading = readFromChannel(channel, bb);
- if (bb.position() > 0) {
- bb.flip();
- tokenizeFragments(bb.asReadOnlyBuffer(), offset, future);
- offset += bb.limit();
- }
- }
- listOfLists = future.getResult();
- List<FileFragment> fragments =
- listOfLists.stream().flatMap(List::stream).collect(Collectors.toList());
-
- assembler.wrapUp(fragments);
- }
-
- private boolean readFromChannel(ReadableByteChannel ch, ByteBuffer bb) throws IOException {
- // Continue reading until we filled the minimum buffer size.
- while (bb.position() < parameters.getMinReadBlockSize()) {
- // Stop if we reached the end of stream.
- if (ch.read(bb) < 0) {
- return false;
- }
- }
- return true;
- }
-
- private void tokenizeFragments(
- ByteBuffer bb,
- long offset,
- CollectingListFuture<List<FileFragment>, GenericParsingException> future) {
- int from = 0;
- int blockSize = parameters.getTokenizeBlockSize();
- while (from < bb.limit()) {
- int to = Math.min(bb.limit(), from + blockSize);
- if (bb.limit() - to < BlockParameters.MIN_TOKENIZE_BLOCK_SIZE) {
- // Do not create the last block too small, rather join it with the previous block.
- to = bb.limit();
- }
- DeclarationConsumer consumer = tokenConsumerFactory.get();
- FileFragment fragment = new FileFragment(bb, offset, from, to);
- FileFragmentSplitter tokenizer = new FileFragmentSplitter(fragment, consumer);
- future.add(executorService.submit(tokenizer));
- from = to;
- }
- }
-
- /** Sizes of blocks for reading from file and parsing for {@link ParallelFileProcessing}. */
- public static class BlockParameters {
-
- private static final int READ_BLOCK_SIZE = 25 * 1024 * 1024;
- private static final int MIN_READ_BLOCK_SIZE = 10 * 1024 * 1024;
- private static final int TOKENIZE_BLOCK_SIZE = 1024 * 1024;
- private static final int MIN_TOKENIZE_BLOCK_SIZE = 100;
-
- /** Size of the reading buffer. */
- private int readBlockSize;
- /**
- * Minimum size of the reading buffer - read() calls will be repeated until the reading buffer
- * has at least minReadBlockSize bytes, only after that the contents will be passed for
- * tokenization.
- */
- private int minReadBlockSize;
- /**
- * Size of the piece for the tokenization task. The read buffer will be split into pieces of
- * tokenizeBlockSize size, and passed for tokenization in parallel.
- */
- private int tokenizeBlockSize;
-
- /** @param fileSize size of the file we are going to parse */
- public BlockParameters(long fileSize) {
- readBlockSize = (int) Math.min(READ_BLOCK_SIZE, fileSize);
- minReadBlockSize = Math.min(MIN_READ_BLOCK_SIZE, (int) Math.ceil((double) fileSize / 2));
- tokenizeBlockSize =
- Math.max(MIN_TOKENIZE_BLOCK_SIZE, Math.min(TOKENIZE_BLOCK_SIZE, minReadBlockSize / 4));
- }
-
- public int getReadBlockSize() {
- return readBlockSize;
- }
-
- /**
- * Sets the size of readBlockSize and adjusts other block sizes so that they together make
- * sense.
- */
- public BlockParameters setReadBlockSize(int readBlockSize) {
- if (readBlockSize > 0) {
- this.readBlockSize = readBlockSize;
- minReadBlockSize = Math.min(minReadBlockSize, (int) Math.ceil((double) readBlockSize / 2));
- tokenizeBlockSize =
- Math.max(MIN_TOKENIZE_BLOCK_SIZE, Math.min(tokenizeBlockSize, minReadBlockSize / 4));
- }
- return this;
- }
-
- public int getTokenizeBlockSize() {
- return tokenizeBlockSize;
- }
-
- public int getMinReadBlockSize() {
- return minReadBlockSize;
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/BUILD
deleted file mode 100644
index 275555a..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/BUILD
+++ /dev/null
@@ -1,22 +0,0 @@
-load("@rules_java//java:defs.bzl", "java_library")
-
-package(
- default_visibility = ["//src:__subpackages__"],
-)
-
-filegroup(
- name = "srcs",
- srcs = glob(["*"]),
- visibility = ["//src:__subpackages__"],
-)
-
-java_library(
- name = "lexer",
- srcs = glob(["*.java"]),
- deps = [
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file",
- "//src/main/java/com/google/devtools/build/lib/util",
- "//third_party:guava",
- "//third_party:jsr305",
- ],
-)
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexer.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexer.java
deleted file mode 100644
index 301be64..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexer.java
+++ /dev/null
@@ -1,254 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.lexer;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import com.google.devtools.build.lib.util.Pair;
-import java.util.Arrays;
-import java.util.List;
-import java.util.stream.Stream;
-
-/** Ninja files lexer. The types of tokens: {@link NinjaToken}. */
-public class NinjaLexer {
- // They all are having different first letter, let's use it.
- private static final ImmutableMap<Byte, NinjaToken> KEYWORD_MAP =
- // There is no #of() method for 6 key-value pairs.
- Stream.of(
- NinjaToken.BUILD,
- NinjaToken.RULE,
- NinjaToken.DEFAULT,
- NinjaToken.SUBNINJA,
- NinjaToken.INCLUDE,
- NinjaToken.POOL)
- .collect(ImmutableMap.toImmutableMap(token -> token.getBytes()[0], nt -> nt));
-
- private final FileFragment fragment;
- private NinjaLexerStep step;
- private final List<Pair<Integer, Integer>> ranges;
- private final List<NinjaToken> tokens;
- /** Flag to give a hint how letters should be interpreted (as text, identifier, path). */
- private TextKind expectedTextKind = TextKind.IDENTIFIER;
-
- private boolean interpretPoolAsVariable = false;
-
- /** @param fragment fragment to do the lexing on */
- public NinjaLexer(FileFragment fragment) {
- this.fragment = fragment;
- step = new NinjaLexerStep(fragment, 0);
- ranges = Lists.newArrayList();
- tokens = Lists.newArrayList();
- }
-
- /**
- * Returns true if following nextToken() call may produce meaningful token. However, it may happen
- * that nextToken() will only produce {@link NinjaToken#EOF}, {@link NinjaToken#ZERO} or {@link
- * NinjaToken#ERROR}.
- *
- * <p>It is an optimization here to check for 'seen' flags: nextToken() may return some meaningful
- * token, and at the same time already discover the end of file or zero byte.
- */
- public boolean hasNextToken() {
- return step.canAdvance();
- }
-
- /**
- * Returns {@link NinjaToken} type of the token for the next non-space and non-comment token
- * at/after current <code>position</code> position.
- */
- public NinjaToken nextToken() {
- Preconditions.checkState(step.canAdvance());
- while (step.canAdvance()) {
- // First byte is checked right in constructor.
- if (step.isSeenZero()) {
- return push(NinjaToken.ZERO);
- }
- byte b = step.startByte();
- switch (b) {
- case ' ':
- case '\t':
- step.skipSpaces();
- if (step.getPosition() == 0
- || NinjaToken.NEWLINE.equals(Iterables.getLast(tokens, null))) {
- return push(NinjaToken.INDENT);
- }
- // Save trailing whitespace as text in case it is meaningful.
- if (step.getEnd() == fragment.length()) {
- return push(NinjaToken.TEXT);
- }
- // Also treat any whitespace before a new line as trailing whitespace, and save it.
- if (step.getEnd() < fragment.length()) {
- byte nextByte = fragment.byteAt(step.getEnd());
- if (nextByte == '\r' || nextByte == '\n') {
- return push(NinjaToken.TEXT);
- }
- }
- break;
- case '\r':
- expectedTextKind = TextKind.IDENTIFIER;
- step.processLineFeedNewLine();
- return push(NinjaToken.NEWLINE);
- case '\n':
- expectedTextKind = TextKind.IDENTIFIER;
- return push(NinjaToken.NEWLINE);
- case '#':
- step.skipComment();
- break;
- case '=':
- if (TextKind.TEXT.equals(expectedTextKind)) {
- step.readText();
- return push(NinjaToken.TEXT);
- }
- return push(NinjaToken.EQUALS);
- case ':':
- return push(NinjaToken.COLON);
- case '|':
- if (TextKind.TEXT.equals(expectedTextKind)) {
- step.readText();
- return push(NinjaToken.TEXT);
- }
- if (step.tryReadDoublePipe()) {
- return push(NinjaToken.PIPE2);
- }
- if (step.tryReadPipeAt()) {
- return push(NinjaToken.PIPE_AT);
- }
- return push(NinjaToken.PIPE);
- case '$':
- if (step.trySkipEscapedNewline()) {
- break;
- }
- if (step.tryReadVariableInBrackets() || step.tryReadSimpleVariable()) {
- return push(NinjaToken.VARIABLE);
- }
- if (step.tryReadEscapedLiteral()) {
- return push(NinjaToken.ESCAPED_TEXT);
- }
- step.forceError("Bad $-escape (literal $ must be written as $$)");
- return push(NinjaToken.ERROR);
- default:
- switch (expectedTextKind) {
- case TEXT:
- step.readText();
- return push(NinjaToken.TEXT);
- case PATH:
- step.readPath();
- return push(NinjaToken.TEXT);
- case IDENTIFIER:
- step.tryReadIdentifier();
- if (step.getError() == null) {
- byte[] bytes = step.getBytes();
- NinjaToken keywordToken = KEYWORD_MAP.get(bytes[0]);
- if (keywordToken != null
- && !(interpretPoolAsVariable && NinjaToken.POOL.equals(keywordToken))
- && Arrays.equals(keywordToken.getBytes(), bytes)) {
- return push(keywordToken);
- }
- }
- return push(NinjaToken.IDENTIFIER);
- }
- throw new IllegalStateException();
- }
- if (step.canAdvance()) {
- step.ensureEnd();
- // For all skipping cases: move to the next step.
- step = step.nextStep();
- }
- }
- // Since we now capture trailing whitespace, this EOF may be unreachable.
- return push(NinjaToken.EOF);
- }
-
- /** Return the bytes of the token, returned by previous nextToken() call. */
- public byte[] getTokenBytes() {
- if (ranges.isEmpty()) {
- throw new IllegalStateException();
- }
- return fragment.getBytes(getLastStart(), getLastEnd());
- }
-
- private NinjaToken push(NinjaToken token) {
- step.ensureEnd();
- ranges.add(Pair.of(step.getStart(), step.getEnd()));
- tokens.add(token);
- if (step.getError() != null) {
- // Do not move in case of error.
- return NinjaToken.ERROR;
- }
- if (step.canAdvance()) {
- step = step.nextStep();
- }
- return token;
- }
-
- public boolean haveReadAnyTokens() {
- return !ranges.isEmpty();
- }
-
- public int getLastStart() {
- if (ranges.isEmpty()) {
- throw new IllegalStateException();
- }
- return Preconditions.checkNotNull(Iterables.getLast(ranges).getFirst());
- }
-
- public int getLastEnd() {
- if (ranges.isEmpty()) {
- throw new IllegalStateException();
- }
- return Preconditions.checkNotNull(Iterables.getLast(ranges).getSecond());
- }
-
- /** Give a hint how letters should be interpreted (as text, identifier, path). */
- public void setExpectedTextKind(TextKind expectedTextKind) {
- this.expectedTextKind = expectedTextKind;
- }
-
- /** When the keyword 'pool' is met, interpret it as identifier, not as a keyword. */
- public void interpretPoolAsVariable() {
- this.interpretPoolAsVariable = true;
- }
-
- /** Undo the previously read token. */
- public void undo() {
- Preconditions.checkState(ranges.size() == tokens.size());
- ranges.remove(ranges.size() - 1);
- tokens.remove(tokens.size() - 1);
- step = new NinjaLexerStep(fragment, ranges.isEmpty() ? 0 : getLastEnd());
- expectedTextKind = TextKind.IDENTIFIER;
- }
-
- public String getError() {
- return step.getError();
- }
-
- public FileFragment getFragment() {
- return fragment;
- }
-
- /**
- * Enum with variants of text fragments parsing: as identifier (most restricted set of symbols),
- * path (all spaces should be $-escaped, and | symbol has a special meaning), or text.
- */
- public enum TextKind {
- IDENTIFIER,
- PATH,
- TEXT
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexerStep.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexerStep.java
deleted file mode 100644
index a697448..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaLexerStep.java
+++ /dev/null
@@ -1,348 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.lexer;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import java.nio.charset.StandardCharsets;
-import java.util.function.Predicate;
-
-/**
- * Helper class for {@link NinjaLexer}. Contains methods for reading Ninja tokens.
- *
- * <p>Start position for reading is fixed. Mutable state includes the end position, offset in case
- * of escaped symbol, error text if a lexing error occurred, and the flag indicating if zero byte
- * was read. (Zero byte determines the end of the file.)
- *
- * <p>Intended to be used like: <code>
- * NinjaLexerStep step = new NinjaLexerStep(fragment, 0);
- * while (step.hasNext()) {
- * byte b = step.startByte();
- * // if/switch, then:
- * step.skipXXX();
- * // or
- * step.tryXXX();
- * // read the end position and error text
- * if (step.hasNext()) {
- * step = nextStep();
- * }
- * }
- * </code>
- */
-public class NinjaLexerStep {
- private static final ImmutableSortedSet<Byte> IDENTIFIER_SYMBOLS =
- createByteSet("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-");
- private static final byte[] TEXT_STOPPERS = createByteArray("\n\r \t$:\u0000");
- // We allow # symbol in the path, so the comment on the line with path can only start with space.
- private static final byte[] PATH_STOPPERS = createByteArray("\n\r \t$:|\u0000");
-
- private static byte[] createByteArray(String variants) {
- byte[] bytes = variants.getBytes(StandardCharsets.ISO_8859_1);
- return bytes;
- }
-
- private static ImmutableSortedSet<Byte> createByteSet(String variants) {
- ImmutableSortedSet.Builder<Byte> builder = ImmutableSortedSet.naturalOrder();
- byte[] bytes = variants.getBytes(StandardCharsets.ISO_8859_1);
- for (byte b : bytes) {
- builder.add(b);
- }
- return builder.build();
- }
-
- private final FileFragment fragment;
- private final int position;
-
- private boolean seenZero;
- private String error;
- private int end;
-
- /**
- * @param position start of the step inside a fragment; must point to a symbol inside fragment.
- */
- public NinjaLexerStep(FileFragment fragment, int position) {
- Preconditions.checkState(position < fragment.length());
- this.fragment = fragment;
- this.position = position;
- end = -1;
- seenZero = position < fragment.length() && (0 == fragment.byteAt(position));
- }
-
- public byte startByte() {
- return fragment.byteAt(position);
- }
-
- public NinjaLexerStep nextStep() {
- Preconditions.checkState(error == null);
- Preconditions.checkState(!seenZero);
-
- return new NinjaLexerStep(fragment, end);
- }
-
- /**
- * Returns true, if there are still symbols to process, i.e. either the next step can be
- * constructed, or if current step was just created, so its bounds are not known yet.
- */
- public boolean canAdvance() {
- return !seenZero && error == null && end < fragment.length();
- }
-
- public FileFragment getFragment() {
- return fragment;
- }
-
- /** Return step bytes, taking into account possible escaped symbol offset. */
- public byte[] getBytes() {
- return fragment.getBytes(position, end);
- }
-
- public int getPosition() {
- return position;
- }
-
- public boolean isSeenZero() {
- return seenZero;
- }
-
- public String getError() {
- return error;
- }
-
- public int getStart() {
- return position;
- }
-
- public int getEnd() {
- return end;
- }
-
- private boolean checkForward(int steps, char... chars) {
- if ((position + steps) < fragment.length()) {
- for (char ch : chars) {
- if ((byte) ch == fragment.byteAt(position + steps)) {
- return true;
- }
- }
- }
- return false;
- }
-
- public void forceError(String error) {
- this.error = error;
- end = position + 1;
- }
-
- public void skipSpaces() {
- end = eatSequence(position, aByte -> ' ' != aByte && '\t' != aByte);
- }
-
- public void skipComment() {
- Preconditions.checkState('#' == fragment.byteAt(position));
- end = eatSequence(position + 1, aByte -> '\n' == aByte || '\r' == aByte);
- }
-
- public boolean trySkipEscapedNewline() {
- Preconditions.checkState('$' == fragment.byteAt(position));
- if (checkForward(1, '\n')) {
- end = position + 2;
- return true;
- } else if (checkForward(1, '\r')) {
- if (checkForward(2, '\n')) {
- end = position + 3;
- } else {
- error = "Wrong newline separators: \\r should be followed by \\n.";
- end = safeEnd(position + 3);
- }
- return true;
- }
- return false;
- }
-
- public void processLineFeedNewLine() {
- Preconditions.checkState('\r' == fragment.byteAt(position));
- if (checkForward(1, '\n')) {
- end = position + 2;
- } else {
- error = "Wrong newline separators: \\r should be followed by \\n.";
- end = safeEnd(position + 2);
- }
- }
-
- public boolean tryReadVariableInBrackets() {
- Preconditions.checkState('$' == fragment.byteAt(position));
- if (checkForward(1, '{')) {
- end = eatSequence(position + 2, aByte -> ' ' != aByte);
- int endOfVariableName = readIdentifier(end, true);
- if (endOfVariableName == end) {
- error = "Variable identifier expected.";
- // Up to the 'wrong' symbol.
- end = endOfVariableName + 1;
- } else {
- end = eatSequence(endOfVariableName, aByte -> ' ' != aByte);
- if (end >= fragment.length() || '}' != fragment.byteAt(end)) {
- error = "Variable end symbol '}' expected.";
- end = safeEnd(end + 1);
- } else {
- ++end;
- }
- }
- return true;
- }
- return false;
- }
-
- public boolean tryReadSimpleVariable() {
- Preconditions.checkState('$' == fragment.byteAt(position));
- if (position + 1 < fragment.length()
- && IDENTIFIER_SYMBOLS.contains(fragment.byteAt(position + 1))) {
- end = readIdentifier(position + 1, false);
- return true;
- }
- return false;
- }
-
- public boolean tryReadEscapedLiteral() {
- Preconditions.checkState('$' == fragment.byteAt(position));
- if (checkForward(1, '$', ':', ' ')) {
- // Escaped literal.
- end = position + 2;
- return true;
- }
- return false;
- }
-
- public void tryReadIdentifier() {
- end = readIdentifier(position, true);
- if (position >= end) {
- error =
- String.format(
- "Symbol '%s' is not allowed in the identifier,"
- + " the text fragment with the symbol:\n%s\n",
- fragment.subFragment(position, position + 1), fragment.getFragmentAround(position));
- end = position + 1;
- }
- }
-
- public boolean tryReadDoublePipe() {
- Preconditions.checkState('|' == fragment.byteAt(position));
- if (checkForward(1, '|')) {
- end = position + 2;
- return true;
- }
- return false;
- }
-
- public boolean tryReadPipeAt() {
- Preconditions.checkState('|' == fragment.byteAt(position));
- if (checkForward(1, '@')) {
- end = position + 2;
- return true;
- }
- return false;
- }
-
- public void readText() {
- int i = position;
- for (; i < fragment.length(); i++) {
- byte b = fragment.byteAt(i);
- if (0 == b) {
- seenZero = true;
- end = i;
- return;
- }
- if (isTextStopper(b)) {
- break;
- }
- }
- end = i;
- }
-
- public void readPath() {
- int i = position;
- for (; i < fragment.length(); i++) {
- byte b = fragment.byteAt(i);
- if (0 == b) {
- seenZero = true;
- end = i;
- return;
- }
- if (isPathStopper(b)) {
- break;
- }
- }
- end = i;
- }
-
- // Optimized, since this is run for each byte in the ninja file. (This has better performance
- // than lookup in a java Set, since TEXT_STOPPERS is small.
- private static boolean isTextStopper(byte b) {
- for (int i = 0; i < TEXT_STOPPERS.length; i++) {
- if (b == TEXT_STOPPERS[i]) {
- return true;
- }
- }
- return false;
- }
-
- // Optimized, since this is run for each byte in the ninja file. (This has better performance
- // than lookup in a java Set, since PATH_STOPPERS is small.
- private static boolean isPathStopper(byte b) {
- for (int i = 0; i < PATH_STOPPERS.length; i++) {
- if (b == PATH_STOPPERS[i]) {
- return true;
- }
- }
- return false;
- }
-
- private int readIdentifier(int startFrom, boolean withDot) {
- if (withDot) {
- return eatSequence(startFrom, b -> !IDENTIFIER_SYMBOLS.contains(b) && '.' != b);
- } else {
- return eatSequence(startFrom, b -> !IDENTIFIER_SYMBOLS.contains(b));
- }
- }
-
- private int safeEnd(int number) {
- return Math.min(fragment.length(), number);
- }
-
- private int eatSequence(int startFrom, Predicate<Byte> stop) {
- int i = startFrom;
- for (; i < fragment.length(); i++) {
- byte b = fragment.byteAt(i);
- if (0 == b) {
- seenZero = true;
- return i;
- }
- if (stop.test(b)) {
- break;
- }
- }
- return i;
- }
-
- /**
- * For the quick checks outside of skipXXX and tryXXX methods of this class, assume that the step
- * takes just one symbol.
- */
- public void ensureEnd() {
- if (end < 0) {
- end = position + 1;
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaToken.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaToken.java
deleted file mode 100644
index b812bef..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer/NinjaToken.java
+++ /dev/null
@@ -1,56 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.lexer;
-
-import java.nio.charset.StandardCharsets;
-import javax.annotation.Nullable;
-
-/** Token types for {@link NinjaLexer}. */
-public enum NinjaToken {
- ERROR("error"),
- BUILD("build"),
- RULE("rule"),
- ESCAPED_TEXT("escaped text symbol"),
- TEXT("text"),
- IDENTIFIER("identifier"),
- VARIABLE("variable"),
- DEFAULT("default"),
- POOL("pool"),
- SUBNINJA("subninja"),
- INCLUDE("include"),
-
- COLON(":"),
- EQUALS("="),
- PIPE("|"),
- PIPE2("||"),
- PIPE_AT("|@"),
-
- INDENT("indent"),
- NEWLINE("newline"),
-
- ZERO("zero byte"),
- EOF("end of file");
-
- private final byte[] bytes;
-
- NinjaToken(@Nullable String text) {
- this.bytes = text != null ? text.getBytes(StandardCharsets.ISO_8859_1) : new byte[0];
- }
-
- public byte[] getBytes() {
- return bytes;
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/BUILD
deleted file mode 100644
index 056f5cd..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/BUILD
+++ /dev/null
@@ -1,41 +0,0 @@
-load("@rules_java//java:defs.bzl", "java_library")
-
-package(
- default_visibility = ["//src:__subpackages__"],
-)
-
-filegroup(
- name = "srcs",
- srcs = glob(["*"]),
- visibility = ["//src:__subpackages__"],
-)
-
-java_library(
- name = "parser",
- srcs = ["NinjaParser.java"],
- deps = [
- ":parser_impl",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline",
- "//src/main/java/com/google/devtools/build/lib/util",
- ],
-)
-
-java_library(
- name = "parser_impl",
- srcs = glob(
- ["*.java"],
- exclude = ["NinjaParser.java"],
- ),
- deps = [
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer",
- "//src/main/java/com/google/devtools/build/lib/collect",
- "//src/main/java/com/google/devtools/build/lib/util",
- "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
- "//third_party:error_prone_annotations",
- "//third_party:guava",
- "//third_party:jsr305",
- ],
-)
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaFileParseResult.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaFileParseResult.java
deleted file mode 100644
index cd4a542..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaFileParseResult.java
+++ /dev/null
@@ -1,201 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.parser;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.util.Pair;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-
-/**
- * Class to hold information about different declarations in Ninja file during parsing.
- *
- * <p>Included files (with include or subninja statements) are kept in form of promises: {@link
- * NinjaPromise<NinjaFileParseResult>}, since for their parsing we may need to first resolve the
- * variables in the current file.
- */
-public class NinjaFileParseResult {
- /**
- * Interface for getting the result of lazy parsing of Ninja file in the context of {@link
- * NinjaScope}.
- *
- * @param <T> result of parsing.
- */
- public interface NinjaPromise<T> {
- T compute(NinjaScope scope) throws GenericParsingException, InterruptedException, IOException;
- }
-
- /** Interface for getting result of lazy parsing of Ninja declaration. */
- public interface NinjaCallable {
- void call() throws GenericParsingException, InterruptedException, IOException;
- }
-
- private final NavigableMap<String, List<Pair<Long, NinjaVariableValue>>> variables;
- private final NavigableMap<String, List<Pair<Long, NinjaRule>>> rules;
- private final NavigableMap<String, List<Pair<Long, NinjaPool>>> pools;
- private final List<FileFragment> targets;
- private final NavigableMap<Long, NinjaPromise<NinjaFileParseResult>> includedFilesFutures;
- private final NavigableMap<Long, NinjaPromise<NinjaFileParseResult>> subNinjaFilesFutures;
-
- public NinjaFileParseResult() {
- variables = Maps.newTreeMap();
- rules = Maps.newTreeMap();
- pools = Maps.newTreeMap();
- targets = Lists.newArrayList();
- includedFilesFutures = Maps.newTreeMap();
- subNinjaFilesFutures = Maps.newTreeMap();
- }
-
- public void addIncludeScope(long offset, NinjaPromise<NinjaFileParseResult> promise) {
- includedFilesFutures.put(offset, promise);
- }
-
- public void addSubNinjaScope(long offset, NinjaPromise<NinjaFileParseResult> promise) {
- subNinjaFilesFutures.put(offset, promise);
- }
-
- public void addTarget(FileFragment fragment) {
- targets.add(fragment);
- }
-
- public void addVariable(String name, long offset, NinjaVariableValue value) {
- variables.computeIfAbsent(name, k -> Lists.newArrayList()).add(Pair.of(offset, value));
- }
-
- public void addRule(long offset, NinjaRule rule) {
- rules.computeIfAbsent(rule.getName(), k -> Lists.newArrayList()).add(Pair.of(offset, rule));
- }
-
- public void addPool(long offset, NinjaPool pool) {
- pools.computeIfAbsent(pool.getName(), k -> Lists.newArrayList()).add(Pair.of(offset, pool));
- }
-
- @VisibleForTesting
- public Map<String, List<Pair<Long, NinjaVariableValue>>> getVariables() {
- return variables;
- }
-
- @VisibleForTesting
- public Map<String, List<Pair<Long, NinjaRule>>> getRules() {
- return rules;
- }
-
- public List<FileFragment> getTargets() {
- return targets;
- }
-
- @VisibleForTesting
- public void sortResults() {
- for (List<Pair<Long, NinjaVariableValue>> list : variables.values()) {
- list.sort(Comparator.comparing(Pair::getFirst));
- }
- for (List<Pair<Long, NinjaRule>> list : rules.values()) {
- list.sort(Comparator.comparing(Pair::getFirst));
- }
- for (List<Pair<Long, NinjaPool>> list : pools.values()) {
- list.sort(Comparator.comparing(Pair::getFirst));
- }
- }
-
- public static NinjaFileParseResult merge(Collection<NinjaFileParseResult> parts) {
- NinjaFileParseResult result = new NinjaFileParseResult();
- if (parts.isEmpty()) {
- return result;
- }
- for (NinjaFileParseResult part : parts) {
- for (Map.Entry<String, List<Pair<Long, NinjaVariableValue>>> entry :
- part.variables.entrySet()) {
- String name = entry.getKey();
- result.variables.computeIfAbsent(name, k -> Lists.newArrayList()).addAll(entry.getValue());
- }
- for (Map.Entry<String, List<Pair<Long, NinjaRule>>> entry : part.rules.entrySet()) {
- String name = entry.getKey();
- result.rules.computeIfAbsent(name, k -> Lists.newArrayList()).addAll(entry.getValue());
- }
- for (Map.Entry<String, List<Pair<Long, NinjaPool>>> entry : part.pools.entrySet()) {
- String name = entry.getKey();
- result.pools.computeIfAbsent(name, k -> Lists.newArrayList()).addAll(entry.getValue());
- }
- result.targets.addAll(part.targets);
- result.includedFilesFutures.putAll(part.includedFilesFutures);
- result.subNinjaFilesFutures.putAll(part.subNinjaFilesFutures);
- }
- result.sortResults();
- return result;
- }
-
- /**
- * Recursively expands variables in the Ninja file and all files it includes (and subninja's).
- * Fills in passed {@link NinjaScope} with the expanded variables and rules, and <code>rawTargets
- * </code> - map of NinjaScope to list of fragments with unparsed Ninja targets.
- */
- public void expandIntoScope(NinjaScope scope, Map<NinjaScope, List<FileFragment>> rawTargets)
- throws InterruptedException, GenericParsingException, IOException {
- scope.setRules(rules);
- scope.setPools(pools);
- rawTargets.put(scope, targets);
-
- TreeMap<Long, NinjaCallable> resolvables = Maps.newTreeMap();
- for (Map.Entry<String, List<Pair<Long, NinjaVariableValue>>> entry : variables.entrySet()) {
- String name = entry.getKey();
- for (Pair<Long, NinjaVariableValue> pair : entry.getValue()) {
- Long offset = Preconditions.checkNotNull(pair.getFirst());
- NinjaVariableValue variableValue = Preconditions.checkNotNull(pair.getSecond());
- resolvables.put(
- offset,
- () ->
- scope.addExpandedVariable(
- offset, name, scope.getExpandedValue(offset, variableValue)));
- }
- }
- for (Map.Entry<Long, NinjaPromise<NinjaFileParseResult>> entry :
- includedFilesFutures.entrySet()) {
- Long offset = entry.getKey();
- resolvables.put(
- offset,
- () -> {
- NinjaFileParseResult fileParseResult = entry.getValue().compute(scope);
- NinjaScope includedScope = scope.addIncluded(offset);
- fileParseResult.expandIntoScope(includedScope, rawTargets);
- });
- }
- for (Map.Entry<Long, NinjaPromise<NinjaFileParseResult>> entry :
- subNinjaFilesFutures.entrySet()) {
- Long offset = entry.getKey();
- resolvables.put(
- offset,
- () -> {
- NinjaFileParseResult fileParseResult = entry.getValue().compute(scope);
- NinjaScope subNinjaScope = scope.addSubNinja(entry.getKey());
- fileParseResult.expandIntoScope(subNinjaScope, rawTargets);
- });
- }
-
- for (NinjaCallable ninjaCallable : resolvables.values()) {
- ninjaCallable.call();
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaParser.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaParser.java
deleted file mode 100644
index d67beac..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaParser.java
+++ /dev/null
@@ -1,148 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.parser;
-
-import com.google.devtools.build.lib.bazel.rules.ninja.file.DeclarationConsumer;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer;
-import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaToken;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaFileParseResult.NinjaPromise;
-import com.google.devtools.build.lib.bazel.rules.ninja.pipeline.NinjaPipeline;
-import com.google.devtools.build.lib.util.Pair;
-import java.io.IOException;
-
-/**
- * Ninja parser: an instance will be created per a fragment of Ninja file, to avoid synchronization
- * while parsing independent values.
- *
- * <p>Populates the {@link NinjaFileParseResult} with variables and rules.
- */
-public class NinjaParser implements DeclarationConsumer {
- private final NinjaPipeline pipeline;
- private final NinjaFileParseResult parseResult;
- private final String ninjaFileName;
-
- private static final String UNSUPPORTED_TOKEN_MESSAGE =
- " is an unsupported type of Ninja lexeme for the parser at this state. "
- + "These are unexpected lexemes at this state for the parser, and running into a "
- + "token of this lexeme hints that the parser did not progress the lexer to a "
- + "valid state during a previous parsing step.";
-
- public NinjaParser(
- NinjaPipeline pipeline, NinjaFileParseResult parseResult, String ninjaFileName) {
- this.pipeline = pipeline;
- this.parseResult = parseResult;
- this.ninjaFileName = ninjaFileName;
- }
-
- @Override
- public void declaration(FileFragment fragment) throws GenericParsingException, IOException {
- long offset = fragment.getFragmentOffset();
-
- NinjaLexer lexer = new NinjaLexer(fragment);
- if (!lexer.hasNextToken()) {
- throw new IllegalStateException("Empty fragment passed as declaration.");
- }
- NinjaToken token = lexer.nextToken();
- // Skip possible leading newlines in the fragment for parsing.
- while (lexer.hasNextToken() && NinjaToken.NEWLINE.equals(token)) {
- token = lexer.nextToken();
- }
- if (!lexer.hasNextToken()) {
- // If fragment contained only newlines.
- return;
- }
- long declarationStart = offset + lexer.getLastStart();
- lexer.undo();
- NinjaParserStep parser =
- new NinjaParserStep(lexer, pipeline.getPathFragmentInterner(), pipeline.getNameInterner());
-
- switch (token) {
- case IDENTIFIER:
- Pair<String, NinjaVariableValue> variable = parser.parseVariable();
- parseResult.addVariable(variable.getFirst(), declarationStart, variable.getSecond());
- break;
- case RULE:
- NinjaRule rule = parser.parseNinjaRule();
- parseResult.addRule(declarationStart, rule);
- break;
- case POOL:
- parseResult.addPool(declarationStart, parser.parseNinjaPool());
- break;
- case INCLUDE:
- NinjaVariableValue includeStatement = parser.parseIncludeStatement();
- NinjaPromise<NinjaFileParseResult> includeFuture =
- pipeline.createChildFileParsingPromise(
- includeStatement, declarationStart, ninjaFileName);
- parseResult.addIncludeScope(declarationStart, includeFuture);
- break;
- case SUBNINJA:
- NinjaVariableValue subNinjaStatement = parser.parseSubNinjaStatement();
- NinjaPromise<NinjaFileParseResult> subNinjaFuture =
- pipeline.createChildFileParsingPromise(
- subNinjaStatement, declarationStart, ninjaFileName);
- parseResult.addSubNinjaScope(declarationStart, subNinjaFuture);
- break;
- case BUILD:
- FileFragment targetFragment;
- if (declarationStart == offset) {
- targetFragment = fragment;
- } else {
- // Method subFragment accepts only the offset *inside fragment*.
- // So we should subtract the offset of fragment's buffer in file
- // (fragment.getFileOffset()),
- // and start of fragment inside buffer (fragment.getStartIncl()).
- long fragmentStart =
- declarationStart - fragment.getFileOffset() - fragment.getStartIncl();
-
- // While the absolute offset is typed as long (because of larger ninja files), the
- // fragments are only at most Integer.MAX_VALUE long, so fragmentStart cannot be
- // larger than that. Check this here.
- if (fragmentStart > Integer.MAX_VALUE) {
- throw new GenericParsingException(
- String.format(
- "The fragmentStart value %s is not expected to be larger than max-int, "
- + "since each fragment is at most max-int long.",
- fragmentStart));
- }
- targetFragment = fragment.subFragment((int) fragmentStart, fragment.length());
- }
- parseResult.addTarget(targetFragment);
- break;
- case DEFAULT:
- // Do nothing.
- break;
- case ZERO:
- case EOF:
- return;
- case COLON:
- case EQUALS:
- case ESCAPED_TEXT:
- case INDENT:
- case NEWLINE:
- case PIPE:
- case PIPE2:
- case PIPE_AT:
- case TEXT:
- case VARIABLE:
- throw new UnsupportedOperationException(token.name() + UNSUPPORTED_TOKEN_MESSAGE);
- case ERROR:
- throw new GenericParsingException(lexer.getError());
- // Omit default case on purpose. Explicitly specify *all* NinjaToken enum cases above or the
- // compilation will fail.
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaParserStep.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaParserStep.java
deleted file mode 100644
index 5561c47..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaParserStep.java
+++ /dev/null
@@ -1,566 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.parser;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Ascii;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Interner;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer;
-import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer.TextKind;
-import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaToken;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget.InputKind;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget.InputOutputKind;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget.OutputKind;
-import com.google.devtools.build.lib.util.Pair;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import javax.annotation.Nullable;
-
-/** Ninja files parser. The types of tokens: {@link NinjaToken}. Ninja lexer: {@link NinjaLexer}. */
-public class NinjaParserStep {
-
- private final NinjaLexer lexer;
- private final Interner<PathFragment> pathFragmentInterner;
- private final Interner<String> nameInterner;
-
- public NinjaParserStep(
- NinjaLexer lexer,
- Interner<PathFragment> pathFragmentInterner,
- Interner<String> nameInterner) {
- this.lexer = lexer;
- this.pathFragmentInterner = pathFragmentInterner;
- this.nameInterner = nameInterner;
- }
-
- /** Parses variable at the current lexer position. */
- public Pair<String, NinjaVariableValue> parseVariable() throws GenericParsingException {
- String name = asString(parseExpected(NinjaToken.IDENTIFIER));
- parseExpected(NinjaToken.EQUALS);
-
- parseAndIgnoreWhitespace();
-
- NinjaVariableValue value = parseVariableValue();
- return Pair.of(nameInterner.intern(name), value);
- }
-
- @VisibleForTesting
- public NinjaVariableValue parseVariableValue() throws GenericParsingException {
- return Preconditions.checkNotNull(parseVariableValueImpl(true));
- }
-
- @Nullable
- private NinjaVariableValue parseVariableValueImpl(boolean noValueAsEmpty) {
- NinjaVariableValue.Builder varBuilder = NinjaVariableValue.builder();
- int previous = -1;
- while (lexer.hasNextToken()) {
- lexer.setExpectedTextKind(TextKind.TEXT);
- NinjaToken token = lexer.nextToken();
- if (NinjaToken.VARIABLE.equals(token)) {
- if (previous >= 0) {
- // add space interval between tokens
- varBuilder.addText(
- asString(lexer.getFragment().getBytes(previous, lexer.getLastStart())));
- }
- varBuilder.addVariable(normalizeVariableName(asString(lexer.getTokenBytes())));
- } else if (NinjaToken.TEXT.equals(token)
- || NinjaToken.ESCAPED_TEXT.equals(token)
- || NinjaToken.COLON.equals(token)) {
- // Add text together with the spaces between current and previous token.
- int start = previous >= 0 ? previous : lexer.getLastStart();
- String rawText = asString(lexer.getFragment().getBytes(start, lexer.getLastEnd()));
- varBuilder.addText(unescapeText(rawText));
- } else {
- lexer.undo();
- break;
- }
- previous = lexer.getLastEnd();
- }
- if (previous == -1) {
- // We read no value.
- if (noValueAsEmpty) {
- // Use empty string for value if specified by caller.
- return NinjaVariableValue.createPlainText("");
- }
- // Otherwise, return null to indicate there was no value.
- return null;
- }
- return varBuilder.build();
- }
-
- /**
- * Paths variable is a sequence of text and variable references until space, newline, eof or |
- * symbol.
- */
- @Nullable
- private NinjaVariableValue parsePathVariableValue() {
- NinjaVariableValue.Builder varBuilder = NinjaVariableValue.builder();
- int previous = -1;
- while (lexer.hasNextToken()) {
- lexer.setExpectedTextKind(TextKind.PATH);
- NinjaToken token = lexer.nextToken();
- if (previous >= 0 && lexer.getLastStart() != previous) {
- // no spaces.
- lexer.undo();
- break;
- }
- if (NinjaToken.VARIABLE.equals(token)) {
- varBuilder.addVariable(normalizeVariableName(asString(lexer.getTokenBytes())));
- } else if (NinjaToken.TEXT.equals(token) || NinjaToken.ESCAPED_TEXT.equals(token)) {
- String rawText = asString(lexer.getTokenBytes());
- varBuilder.addText(unescapeText(rawText));
- } else {
- lexer.undo();
- break;
- }
- previous = lexer.getLastEnd();
- }
- if (previous == -1) {
- // We read no value.
- return null;
- }
- return varBuilder.build();
- }
-
- private static String unescapeText(String text) {
- StringBuilder sb = new StringBuilder(text.length());
- for (int i = 0; i < text.length(); i++) {
- char ch = text.charAt(i);
- if (ch == '$') {
- Preconditions.checkState(i + 1 < text.length());
- sb.append(text.charAt(i + 1));
- i++;
- } else {
- sb.append(ch);
- }
- }
- return sb.toString();
- }
-
- public NinjaVariableValue parseIncludeStatement() throws GenericParsingException {
- return parseIncludeOrSubNinja(NinjaToken.INCLUDE);
- }
-
- public NinjaVariableValue parseSubNinjaStatement() throws GenericParsingException {
- return parseIncludeOrSubNinja(NinjaToken.SUBNINJA);
- }
-
- private NinjaVariableValue parseIncludeOrSubNinja(NinjaToken token)
- throws GenericParsingException {
- parseExpected(token);
- parseAndIgnoreWhitespace();
- NinjaVariableValue value = parseVariableValueImpl(false);
- if (value == null) {
- throw new GenericParsingException(
- String.format("%s statement has no path.", Ascii.toLowerCase(token.name())));
- }
- if (lexer.hasNextToken()) {
- parseExpected(NinjaToken.NEWLINE);
- lexer.undo();
- }
- return value;
- }
-
- // Consume and skip over any TEXT with a consecutive sequence of ' ' characters.
- private void parseAndIgnoreWhitespace() {
- while (lexer.hasNextToken()) {
- NinjaToken token = lexer.nextToken();
- if (token != NinjaToken.TEXT) {
- lexer.undo();
- break;
- }
- byte[] bytes = lexer.getTokenBytes();
- boolean foundNonWhitespace = false;
- for (int i = 0; i < bytes.length; i++) {
- if (bytes[i] != ' ') {
- foundNonWhitespace = true;
- break;
- }
- }
- if (foundNonWhitespace) {
- lexer.undo();
- break;
- }
- }
- }
-
- /** Parses Ninja rule at the current lexer position. */
- public NinjaRule parseNinjaRule() throws GenericParsingException {
- parseExpected(NinjaToken.RULE);
- String name = asString(parseExpected(NinjaToken.IDENTIFIER));
- parseAndIgnoreWhitespace();
-
- ImmutableSortedMap.Builder<NinjaRuleVariable, NinjaVariableValue> variablesBuilder =
- ImmutableSortedMap.naturalOrder();
-
- parseExpected(NinjaToken.NEWLINE);
- lexer.interpretPoolAsVariable();
- while (parseIndentOrFinishDeclaration()) {
- String variableName = asString(parseExpected(NinjaToken.IDENTIFIER));
- parseExpected(NinjaToken.EQUALS);
- NinjaVariableValue value = parseVariableValue();
-
- NinjaRuleVariable ninjaRuleVariable = NinjaRuleVariable.nullOrValue(variableName);
- if (ninjaRuleVariable == null) {
- throw new GenericParsingException(String.format("Unexpected variable '%s'", variableName));
- }
- variablesBuilder.put(ninjaRuleVariable, value);
- if (lexer.hasNextToken()) {
- parseExpected(NinjaToken.NEWLINE);
- }
- }
- return new NinjaRule(nameInterner.intern(name), variablesBuilder.buildOrThrow());
- }
-
- /** Parses Ninja pool at the current lexer position. */
- public NinjaPool parseNinjaPool() throws GenericParsingException {
- // TODO: consider using generics to condense this and parseNinjaRule into the same
- // method body.
- parseExpected(NinjaToken.POOL);
- String name = asString(parseExpected(NinjaToken.IDENTIFIER));
-
- ImmutableSortedMap.Builder<NinjaPoolVariable, NinjaVariableValue> variablesBuilder =
- ImmutableSortedMap.naturalOrder();
-
- parseExpected(NinjaToken.NEWLINE);
- while (parseIndentOrFinishDeclaration()) {
- String variableName = asString(parseExpected(NinjaToken.IDENTIFIER));
- parseExpected(NinjaToken.EQUALS);
- NinjaVariableValue value = parseVariableValue();
-
- NinjaPoolVariable ninjaPoolVariable = NinjaPoolVariable.nullOrValue(variableName);
- if (ninjaPoolVariable == null) {
- throw new GenericParsingException(String.format("Unexpected variable '%s'", variableName));
- }
- variablesBuilder.put(ninjaPoolVariable, value);
- if (lexer.hasNextToken()) {
- parseExpected(NinjaToken.NEWLINE);
- }
- }
- return new NinjaPool(nameInterner.intern(name), variablesBuilder.buildOrThrow());
- }
-
- private enum NinjaTargetParsingPart {
-
- OUTPUTS(OutputKind.EXPLICIT, true),
- IMPLICIT_OUTPUTS(OutputKind.IMPLICIT, true),
- INPUTS(InputKind.EXPLICIT, false),
- IMPLICIT_INPUTS(InputKind.IMPLICIT, false),
- ORDER_ONLY_INPUTS(InputKind.ORDER_ONLY, false),
- VALIDATION_INPUTS(InputKind.VALIDATION, false),
- RULE_NAME(null, false),
- VARIABLES(null, false);
-
- @Nullable private final InputOutputKind inputOutputKind;
- private final boolean transitionRequired;
-
- NinjaTargetParsingPart(@Nullable InputOutputKind inputOutputKind, boolean transitionRequired) {
- this.inputOutputKind = inputOutputKind;
- this.transitionRequired = transitionRequired;
- }
-
- @Nullable
- public InputOutputKind getInputOutputKind() {
- return inputOutputKind;
- }
-
- public boolean isTransitionRequired() {
- return transitionRequired;
- }
- }
-
- /**
- * Mapping for changing the {@link NinjaTargetParsingPart} according to the next separator symbol.
- */
- private static final ImmutableSortedMap<
- NinjaTargetParsingPart, ImmutableSortedMap<NinjaToken, NinjaTargetParsingPart>>
- TARGET_PARTS_TRANSITIONS_MAP =
- ImmutableSortedMap
- .<NinjaTargetParsingPart, ImmutableSortedMap<NinjaToken, NinjaTargetParsingPart>>
- naturalOrder()
- .put(
- NinjaTargetParsingPart.OUTPUTS,
- ImmutableSortedMap.of(
- NinjaToken.PIPE, NinjaTargetParsingPart.IMPLICIT_OUTPUTS,
- NinjaToken.COLON, NinjaTargetParsingPart.RULE_NAME))
- .put(
- NinjaTargetParsingPart.IMPLICIT_OUTPUTS,
- ImmutableSortedMap.of(NinjaToken.COLON, NinjaTargetParsingPart.RULE_NAME))
- // Because there is no specific token separating the rule name from the inputs
- // (besides a space), there is no entry for transitioning to INPUTS, and transitioning
- // is instead handled in parseTargetDependenciesPart().
- .put(
- NinjaTargetParsingPart.INPUTS,
- ImmutableSortedMap.of(
- NinjaToken.PIPE, NinjaTargetParsingPart.IMPLICIT_INPUTS,
- NinjaToken.PIPE2, NinjaTargetParsingPart.ORDER_ONLY_INPUTS,
- NinjaToken.PIPE_AT, NinjaTargetParsingPart.VALIDATION_INPUTS,
- NinjaToken.NEWLINE, NinjaTargetParsingPart.VARIABLES))
- .put(
- NinjaTargetParsingPart.IMPLICIT_INPUTS,
- ImmutableSortedMap.of(
- NinjaToken.PIPE2, NinjaTargetParsingPart.ORDER_ONLY_INPUTS,
- NinjaToken.PIPE_AT, NinjaTargetParsingPart.VALIDATION_INPUTS,
- NinjaToken.NEWLINE, NinjaTargetParsingPart.VARIABLES))
- .put(
- NinjaTargetParsingPart.ORDER_ONLY_INPUTS,
- ImmutableSortedMap.of(
- NinjaToken.PIPE_AT, NinjaTargetParsingPart.VALIDATION_INPUTS,
- NinjaToken.NEWLINE, NinjaTargetParsingPart.VARIABLES))
- .put(
- NinjaTargetParsingPart.VALIDATION_INPUTS,
- ImmutableSortedMap.of(NinjaToken.NEWLINE, NinjaTargetParsingPart.VARIABLES))
- .build();
-
- /**
- * Parses Ninja target using {@link NinjaScope} of the file, where it is defined, to expand
- * variables.
- */
- public NinjaTarget parseNinjaTarget(NinjaScope fileScope, long offset)
- throws GenericParsingException {
- NinjaTarget.Builder builder = NinjaTarget.builder(fileScope, offset, nameInterner);
- parseExpected(NinjaToken.BUILD);
-
- Map<InputOutputKind, List<NinjaVariableValue>> pathValuesMap =
- parseTargetDependenciesPart(builder);
-
- NinjaScope targetScope = parseTargetVariables(offset, fileScope, builder);
-
- // Variables from the build statement can be used in the input and output paths, so
- // we are using targetScope to resolve paths values.
- for (Map.Entry<InputOutputKind, List<NinjaVariableValue>> entry : pathValuesMap.entrySet()) {
- List<PathFragment> paths =
- entry.getValue().stream()
- .map(
- value ->
- pathFragmentInterner.intern(
- PathFragment.create(targetScope.getExpandedValue(Long.MAX_VALUE, value))))
- .collect(Collectors.toList());
- InputOutputKind inputOutputKind = entry.getKey();
- if (inputOutputKind instanceof InputKind) {
- builder.addInputs((InputKind) inputOutputKind, paths);
- } else {
- builder.addOutputs((OutputKind) inputOutputKind, paths);
- }
- }
-
- return builder.build();
- }
-
- /**
- * We resolve build statement variables values, using the file scope: build statement variable
- * values can not refer to each other. Then we are constructing the target's {@link NinjaScope}
- * with already expanded variables; it will be used for resolving target's input and output paths
- * (which can also refer to file-level variables, so we better reuse resolve logic that we already
- * have in NinjaScope).
- *
- * <p>As we expand variable values, we are adding them to {@link NinjaTarget.Builder}.
- *
- * <p>Ninja targets can not refer to the rule's variables values, because the rule variables are
- * only expanded when the rule is used, and the rule is used for already parsed target. However,
- * target's variables can override values of rule's variables.
- *
- * @return Ninja scope for expanding input and output paths of that statement
- */
- private NinjaScope parseTargetVariables(
- long offset, NinjaScope fileScope, NinjaTarget.Builder builder)
- throws GenericParsingException {
- Map<String, List<Pair<Long, String>>> expandedVariables = Maps.newHashMap();
- lexer.interpretPoolAsVariable();
- while (parseIndentOrFinishDeclaration()) {
- Pair<String, NinjaVariableValue> pair = parseVariable();
- String name = Preconditions.checkNotNull(pair.getFirst());
- NinjaVariableValue value = Preconditions.checkNotNull(pair.getSecond());
- String expandedValue = fileScope.getExpandedValue(offset, value);
- expandedVariables
- .computeIfAbsent(name, k -> Lists.newArrayList())
- .add(Pair.of(0L, expandedValue));
- builder.addVariable(name, expandedValue);
-
- if (lexer.hasNextToken()) {
- parseExpected(NinjaToken.NEWLINE);
- }
- }
- return fileScope.createScopeFromExpandedValues(ImmutableSortedMap.copyOf(expandedVariables));
- }
-
- /**
- * Parse build statement dependencies part: output1..k [| implicit_output1..k]: rule input1..k [|
- * implicit_input1..k] [|| order_only_input1..k]
- */
- private Map<InputOutputKind, List<NinjaVariableValue>> parseTargetDependenciesPart(
- NinjaTarget.Builder builder) throws GenericParsingException {
-
- Map<InputOutputKind, List<NinjaVariableValue>> pathValuesMap = Maps.newHashMap();
- boolean ruleNameParsed = false;
- NinjaTargetParsingPart parsingPart = NinjaTargetParsingPart.OUTPUTS;
-
- while (lexer.hasNextToken() && !NinjaTargetParsingPart.VARIABLES.equals(parsingPart)) {
-
- if (NinjaTargetParsingPart.RULE_NAME.equals(parsingPart)) {
- ruleNameParsed = true;
- builder.setRuleName(asString(parseExpected(NinjaToken.IDENTIFIER)));
- parsingPart = NinjaTargetParsingPart.INPUTS;
- continue;
- }
-
- List<NinjaVariableValue> paths = parsePaths();
- if (paths.isEmpty() && !NinjaTargetParsingPart.INPUTS.equals(parsingPart)) {
- throw new GenericParsingException("Expected paths sequence");
- }
-
- if (!paths.isEmpty()) {
- pathValuesMap.put(Preconditions.checkNotNull(parsingPart.getInputOutputKind()), paths);
- }
-
- if (!lexer.hasNextToken()) {
- if (parsingPart.isTransitionRequired()) {
- throw new GenericParsingException("Unexpected end of target");
- }
- break;
- }
-
- NinjaToken lexicalSeparator = lexer.nextToken();
- parsingPart =
- Preconditions.checkNotNull(TARGET_PARTS_TRANSITIONS_MAP.get(parsingPart))
- .get(lexicalSeparator);
-
- if (parsingPart == null) {
- throw new GenericParsingException("Unexpected token: " + lexicalSeparator);
- }
- }
-
- if (!ruleNameParsed) {
- throw new GenericParsingException("Expected rule name");
- }
- Preconditions.checkState(
- !lexer.hasNextToken() || NinjaTargetParsingPart.VARIABLES.equals(parsingPart));
-
- return pathValuesMap;
- }
-
- private List<NinjaVariableValue> parsePaths() {
- List<NinjaVariableValue> result = Lists.newArrayList();
- NinjaVariableValue value;
- while (lexer.hasNextToken() && (value = parsePathVariableValue()) != null) {
- result.add(value);
- }
- return result;
- }
-
- @VisibleForTesting
- public static String normalizeVariableName(String raw) {
- // We start from 1 because it is always at least $ marker symbol in the beginning
- int start = 1;
- for (; start < raw.length(); start++) {
- char ch = raw.charAt(start);
- if (' ' != ch && '$' != ch && '{' != ch) {
- break;
- }
- }
- int end = raw.length() - 1;
- for (; end > start; end--) {
- char ch = raw.charAt(end);
- if (' ' != ch && '}' != ch) {
- break;
- }
- }
- return raw.substring(start, end + 1);
- }
-
- private static String asString(byte[] value) {
- return new String(value, StandardCharsets.ISO_8859_1);
- }
-
- /**
- * It is expected that indent is preceding to the identifier in the scoped variable declaration.
- * It can be, however, that it is just an empty line with spaces - in that case, we want to
- * interpret it as the finish of the currently parsed lexeme.
- *
- * @return true if indent was parsed and there is something different than the newline after it.
- */
- private boolean parseIndentOrFinishDeclaration() throws GenericParsingException {
- if (!lexer.hasNextToken()) {
- return false;
- }
- NinjaToken token = lexer.nextToken();
- boolean isIndent = NinjaToken.INDENT.equals(token);
- if (!isIndent && !NinjaToken.NEWLINE.equals(token)) {
- throwNotExpectedTokenError(NinjaToken.INDENT, token);
- }
- // Check for indent followed by newline, or end of file.
- if (lexer.hasNextToken()) {
- NinjaToken afterIndent = lexer.nextToken();
- lexer.undo();
- if (NinjaToken.NEWLINE.equals(afterIndent)) {
- return false;
- }
- } else {
- return false;
- }
- return isIndent;
- }
-
- private byte[] parseExpected(NinjaToken expectedToken) throws GenericParsingException {
- checkLexerHasNextToken(expectedToken);
- NinjaToken token = lexer.nextToken();
- if (!expectedToken.equals(token)) {
- throwNotExpectedTokenError(expectedToken, token);
- }
- return lexer.getTokenBytes();
- }
-
- private void throwNotExpectedTokenError(NinjaToken expectedToken, NinjaToken token)
- throws GenericParsingException {
- String actual =
- NinjaToken.ERROR.equals(token)
- ? String.format("error: '%s'", lexer.getError())
- : asString(token.getBytes());
- throw new GenericParsingException(
- String.format(
- "Expected %s, but got %s in fragment:\n%s\n",
- asString(expectedToken.getBytes()),
- actual,
- lexer.getFragment().getFragmentAround(lexer.getLastStart())));
- }
-
- private void checkLexerHasNextToken(NinjaToken expectedToken) throws GenericParsingException {
- if (!lexer.hasNextToken()) {
- String message;
- if (lexer.haveReadAnyTokens()) {
- message =
- String.format(
- "Expected %s after '%s' in fragment:\n%s\n",
- asString(expectedToken.getBytes()),
- asString(lexer.getTokenBytes()),
- lexer.getFragment().getFragmentAround(lexer.getLastStart()));
- } else {
- message =
- String.format(
- "Expected %s, but found no text to parse after fragment:\n%s\n",
- asString(expectedToken.getBytes()),
- lexer.getFragment().getFragmentAround(lexer.getLastStart()));
- }
- throw new GenericParsingException(message);
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaPool.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaPool.java
deleted file mode 100644
index 893515e..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaPool.java
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.parser;
-
-
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import javax.annotation.concurrent.Immutable;
-
-/**
- * Ninja pool representation. A Ninja pool allocates rules to use a finite number of concurrent jobs
- * instead of the default parallelism computed by Ninja.
- *
- * <p>While there is no current logical mapping between pool depths and execution phase resource
- * scheduling, this exists for the completeness of the {@link NinjaParser}, and bringing pool
- * objects into {@link NinjaScope}.
- *
- * <p>{@link NinjaVariableValue} to be replaced for each target according to the scope rules.
- *
- * <p>See <a href="https://ninja-build.org/manual.html#ref_pool">Ninja docs</a> for more info.
- */
-@Immutable
-public final class NinjaPool {
- private final String name;
- private final Integer depth;
-
- public NinjaPool(String name, ImmutableSortedMap<NinjaPoolVariable, NinjaVariableValue> variables)
- throws GenericParsingException {
- this.name = name;
- this.depth = validateDepth(variables.get(NinjaPoolVariable.DEPTH));
- }
-
- /** Returns name of the ninja pool. */
- public String getName() {
- return name;
- }
-
- /** Returns depth of the ninja pool, or the maximum concurrent number of jobs in this pool. */
- public Integer getDepth() {
- return depth;
- }
-
- private static Integer validateDepth(NinjaVariableValue value) throws GenericParsingException {
- String rawValue = value.getRawText();
- try {
- return Integer.parseInt(rawValue);
- } catch (NumberFormatException e) {
- throw new GenericParsingException(
- String.format("Expected an integer for the 'depth' value, but got '%s'.", rawValue), e);
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaPoolVariable.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaPoolVariable.java
deleted file mode 100644
index 28fb6cb..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaPoolVariable.java
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.parser;
-
-import com.google.common.base.Ascii;
-
-/** Enum to represent {@link NinjaPool} variables with the special value, like name or depth. */
-public enum NinjaPoolVariable {
- DEPTH;
-
- public static NinjaPoolVariable nullOrValue(String name) {
- try {
- return valueOf(Ascii.toUpperCase(name));
- } catch (IllegalArgumentException e) {
- return null;
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaRule.java
deleted file mode 100644
index 7f0e0df..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaRule.java
+++ /dev/null
@@ -1,53 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.parser;
-
-import com.google.common.collect.ImmutableSortedMap;
-import javax.annotation.concurrent.Immutable;
-
-/**
- * Ninja rule representation.
- *
- * <p>{@link NinjaVariableValue} to be replaced for each target according to the scope rules.
- */
-@Immutable
-public final class NinjaRule {
- private final ImmutableSortedMap<NinjaRuleVariable, NinjaVariableValue> variables;
- private final String name;
-
- public NinjaRule(
- String name, ImmutableSortedMap<NinjaRuleVariable, NinjaVariableValue> variables) {
- this.name = name;
- this.variables = variables;
- }
-
- public ImmutableSortedMap<NinjaRuleVariable, NinjaVariableValue> getVariables() {
- return variables;
- }
-
- public String getName() {
- return name;
- }
-
- /**
- * Returns the raw description string of the Ninja rule with unexpanded variables. If the rule
- * does not have a description key, return an empty string instead.
- */
- public String getDescription() {
- NinjaVariableValue value = variables.get(NinjaRuleVariable.DESCRIPTION);
- return value == null ? "" : value.getRawText();
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaRuleVariable.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaRuleVariable.java
deleted file mode 100644
index 29c094a..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaRuleVariable.java
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.parser;
-
-import com.google.common.base.Ascii;
-
-/** Enum to represent {@link NinjaRule} variables with the special value, like name or command. */
-public enum NinjaRuleVariable {
- COMMAND,
- DEPFILE,
- DEPS,
- MSVC_DEPS_PREFIX,
- DESCRIPTION,
- GENERATOR,
- RESTAT,
- RSPFILE,
- RSPFILE_CONTENT,
- SYMLINK_OUTPUTS,
- POOL;
-
- public String lowerCaseName() {
- return Ascii.toLowerCase(name());
- }
-
- public static NinjaRuleVariable nullOrValue(String name) {
- try {
- return valueOf(Ascii.toUpperCase(name));
- } catch (IllegalArgumentException e) {
- return null;
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaScope.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaScope.java
deleted file mode 100644
index 0703adf..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaScope.java
+++ /dev/null
@@ -1,282 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.parser;
-
-import static com.google.common.base.Strings.nullToEmpty;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Interner;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.devtools.build.lib.util.Pair;
-import java.util.ArrayDeque;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import javax.annotation.Nullable;
-
-/**
- * Ninja file scope to keep all defined variables and rules according to the order of their
- * definition (and redefinition).
- */
-public class NinjaScope {
- /** Parent scope for the case of subninja/include command */
- @Nullable private final NinjaScope parentScope;
- /** If include command was used for the current scope, the offset of that include command */
- @Nullable private final Long includePoint;
-
- private final NavigableMap<Long, NinjaScope> includedScopes;
- private final NavigableMap<Long, NinjaScope> subNinjaScopes;
- private Map<String, List<Pair<Long, String>>> expandedVariables;
- private final Map<String, List<Pair<Long, NinjaRule>>> rules;
- private final Map<String, List<Pair<Long, NinjaPool>>> pools;
-
- public NinjaScope() {
- this(null, null);
- }
-
- private NinjaScope(@Nullable NinjaScope parentScope, @Nullable Long includePoint) {
- this.parentScope = parentScope;
- this.includePoint = includePoint;
- this.rules = Maps.newTreeMap();
- this.pools = Maps.newTreeMap();
- this.includedScopes = Maps.newTreeMap();
- this.subNinjaScopes = Maps.newTreeMap();
- this.expandedVariables = Maps.newHashMap();
- }
-
- public void setRules(Map<String, List<Pair<Long, NinjaRule>>> rules) {
- this.rules.putAll(rules);
- }
-
- public void setPools(Map<String, List<Pair<Long, NinjaPool>>> pools) {
- this.pools.putAll(pools);
- }
-
- @VisibleForTesting
- public Map<String, List<Pair<Long, NinjaRule>>> getRules() {
- return rules;
- }
-
- @VisibleForTesting
- public Map<String, List<Pair<Long, NinjaPool>>> getPools() {
- return pools;
- }
-
- public Collection<NinjaScope> getIncludedScopes() {
- return includedScopes.values();
- }
-
- public Collection<NinjaScope> getSubNinjaScopes() {
- return subNinjaScopes.values();
- }
-
- /**
- * Expands variable value at the given offset. If some of the variable references, used in the
- * value, can not be found, uses an empty string as their value.
- */
- public String getExpandedValue(long offset, NinjaVariableValue value) {
- // Cache expanded variables values to save time replacing several references to the same
- // variable.
- // This cache is local to the offset, it depends on the offset of the variable we are expanding.
- Map<String, String> cache = Maps.newHashMap();
- // We are using the start offset of the value holding the reference to the variable.
- // Do the same as Ninja implementation: if the variable is not found, use empty string.
- Function<String, String> expander =
- ref -> cache.computeIfAbsent(ref, (key) -> nullToEmpty(findExpandedVariable(offset, key)));
- return value.getExpandedValue(expander);
- }
-
- /**
- * Partially expands variable value at the given offset. If some of the variable references, used
- * in the value, can not be found, they are left unexpanded.
- */
- public NinjaVariableValue getReducedValue(
- long offset,
- NinjaVariableValue value,
- ImmutableSet<String> variablesToNotExpand,
- Interner<String> interner) {
- // Cache expanded variables values to save time replacing several references to the same
- // variable.
- // This cache is local to the offset, it depends on the offset of the variable we are expanding.
- Map<String, String> cache = Maps.newHashMap();
- // We are using the start offset of the value holding the reference to the variable.
- // Do the same as Ninja implementation: if the variable is not found, use empty string.
- Function<String, String> expander =
- ref -> computeExpandedString(offset, ref, cache, variablesToNotExpand, interner);
- return value.reduce(expander);
- }
-
- private String computeExpandedString(
- long offset,
- String key,
- Map<String, String> cache,
- ImmutableSet<String> variablesToNotExpand,
- Interner<String> interner) {
- String cachedValue = cache.get(key);
- if (cachedValue != null) {
- return cachedValue;
- }
- if (variablesToNotExpand.contains(key)) {
- return null;
- }
- String expandedValue = findExpandedVariable(offset, key);
- // It's very important an interner is used here, as there is considerable duplication of
- // string literals in expanded rule-variable-parts over the course of a large build.
- return expandedValue == null ? null : interner.intern(expandedValue);
- }
-
- public void addExpandedVariable(Long offset, String name, String value) {
- expandedVariables.computeIfAbsent(name, k -> Lists.newArrayList()).add(Pair.of(offset, value));
- }
-
- public NinjaScope addIncluded(Long offset) {
- NinjaScope scope = new NinjaScope(this, offset);
- includedScopes.put(offset, scope);
- return scope;
- }
-
- public NinjaScope addSubNinja(Long offset) {
- NinjaScope scope = new NinjaScope(this, offset);
- subNinjaScopes.put(offset, scope);
- return scope;
- }
-
- /**
- * Finds expanded variable with the name <code>name</code> to be used in the reference to it at
- * <code>offset</code>. Returns null if nothing was found.
- */
- @Nullable
- public String findExpandedVariable(long offset, String name) {
- return findByNameAndOffsetRecursively(offset, name, scope -> scope.expandedVariables);
- }
-
- /**
- * Finds a rule with the name <code>name</code> to be used in the reference to it at <code>offset
- * </code>. Returns null if nothing was found.
- */
- @Nullable
- public NinjaRule findRule(long offset, String name) {
- return findByNameAndOffsetRecursively(offset, name, scope -> scope.rules);
- }
-
- /**
- * Finds a variable or rule with the name <code>name</code> to be used in the reference to it at
- * <code>offset</code>.
- *
- * <p>The following checks are made: - the last definition of variable/rule before the offset in
- * the current scope is looked up. - the last definition of variable/rule inside the relevant
- * included scopes (i.e. in the files from include statements before offset)
- *
- * <p>If any of the definitions are found in the current or included scopes, the value with the
- * largest offset is returned.
- *
- * <p>If nothing is found, we make an attempt to find the definition in the parent scope at offset
- * before the offset at which the current scope was introduced to parent.
- *
- * <p>If no definition was found, we return null.
- */
- @Nullable
- private <T> T findByNameAndOffsetRecursively(
- long offset,
- String name,
- Function<NinjaScope, Map<String, List<Pair<Long, T>>>> mapSupplier) {
- Pair<Long, T> currentScopeValue = findByNameAndOffset(offset, name, this, mapSupplier);
-
- Long currentScopeOffset =
- currentScopeValue != null ? Preconditions.checkNotNull(currentScopeValue.getFirst()) : -1L;
-
- // Search in included scopes, which were included after the current scope, so they could
- // override the value, but before the reference offset.
- NavigableMap<Long, NinjaScope> subMap =
- includedScopes.subMap(currentScopeOffset, false, offset, false);
- // Search in descending order, so that the first found value is the result.
- for (NinjaScope includedScope : subMap.descendingMap().values()) {
- T includedValue =
- includedScope.findByNameAndOffsetRecursively(Long.MAX_VALUE, name, mapSupplier);
- if (includedValue != null) {
- return includedValue;
- }
- }
- if (currentScopeValue != null) {
- return currentScopeValue.getSecond();
- }
- if (parentScope != null) {
- Preconditions.checkNotNull(includePoint);
- // -1 is used in order not to conflict with the current scope.
- return parentScope.findByNameAndOffsetRecursively(includePoint - 1, name, mapSupplier);
- }
- return null;
- }
-
- /**
- * Finds the variable or rule with the name <code>name</code>, defined in the current scope before
- * the <code>offset</code>. (Ninja allows to re-define the values of rules and variables.)
- */
- @Nullable
- private static <T> Pair<Long, T> findByNameAndOffset(
- long offset,
- String name,
- NinjaScope scope,
- Function<NinjaScope, Map<String, List<Pair<Long, T>>>> mapFunction) {
- List<Pair<Long, T>> pairs = Preconditions.checkNotNull(mapFunction.apply(scope)).get(name);
- if (pairs == null) {
- // We may want to search in the parent scope.
- return null;
- }
- int insertionPoint =
- Collections.binarySearch(
- pairs, Pair.of(offset, null), Comparator.comparing(Pair::getFirst));
- if (insertionPoint >= 0) {
- // Can not be, variable can not be defined in exactly same place.
- throw new IllegalStateException("Trying to interpret declaration as reference.");
- }
- // We need to access the previous element, before the insertion point.
- int idx = -insertionPoint - 2;
- if (idx < 0) {
- // Check the parent scope.
- return null;
- }
- Pair<Long, T> pair = pairs.get(idx);
- return Pair.of(pair.getFirst(), pair.getSecond());
- }
-
- public NinjaScope createScopeFromExpandedValues(
- ImmutableSortedMap<String, List<Pair<Long, String>>> expandedVariables) {
- NinjaScope scope = new NinjaScope(this, Long.MAX_VALUE);
- scope.expandedVariables.putAll(expandedVariables);
- return scope;
- }
-
- public void iterate(Consumer<NinjaScope> consumer) {
- ArrayDeque<NinjaScope> queue = new ArrayDeque<>();
- queue.add(this);
- while (!queue.isEmpty()) {
- NinjaScope currentScope = queue.removeFirst();
- consumer.accept(currentScope);
- queue.addAll(currentScope.getIncludedScopes());
- queue.addAll(currentScope.getSubNinjaScopes());
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaTarget.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaTarget.java
deleted file mode 100644
index b3678fc..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaTarget.java
+++ /dev/null
@@ -1,361 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.parser;
-
-import com.google.common.base.Ascii;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Interner;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.collect.ImmutableSortedKeyListMultimap;
-import com.google.devtools.build.lib.util.Pair;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.errorprone.annotations.Immutable;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/** Ninja target (build statement) representation. */
-public final class NinjaTarget {
-
- /**
- * A list of variables that pertain to input and output artifact paths of a Ninja target. For
- * memory savings, these variables should be expanded late, and the expansion results not retained
- * in memory. (Otherwise, these paths are persisted twice!)
- */
- private static final ImmutableSet<String> INPUTS_OUTPUTS_VARIABLES =
- ImmutableSet.of("in", "in_newline", "out");
-
- /** Builder for {@link NinjaTarget}. */
- public static class Builder {
- private String ruleName;
- private final ImmutableSortedKeyListMultimap.Builder<InputKind, PathFragment> inputsBuilder;
- private final ImmutableSortedKeyListMultimap.Builder<OutputKind, PathFragment> outputsBuilder;
- private final NinjaScope scope;
- private final long offset;
-
- private final ImmutableSortedMap.Builder<String, String> variablesBuilder;
- private final Interner<String> nameInterner;
-
- private Builder(NinjaScope scope, long offset, Interner<String> nameInterner) {
- this.scope = scope;
- this.offset = offset;
- inputsBuilder = ImmutableSortedKeyListMultimap.builder();
- outputsBuilder = ImmutableSortedKeyListMultimap.builder();
- variablesBuilder = ImmutableSortedMap.naturalOrder();
- this.nameInterner = nameInterner;
- }
-
- public Builder setRuleName(String ruleName) {
- this.ruleName = ruleName;
- return this;
- }
-
- public Builder addInputs(InputKind kind, Collection<PathFragment> inputs) {
- inputsBuilder.putAll(kind, inputs);
- return this;
- }
-
- public Builder addOutputs(OutputKind kind, Collection<PathFragment> outputs) {
- outputsBuilder.putAll(kind, outputs);
- return this;
- }
-
- public Builder addVariable(String key, String value) {
- variablesBuilder.put(key, value);
- return this;
- }
-
- public NinjaTarget build() throws GenericParsingException {
- Preconditions.checkNotNull(ruleName);
- String internedName = nameInterner.intern(ruleName);
- ImmutableSortedMap<String, String> variables = variablesBuilder.buildOrThrow();
-
- ImmutableSortedMap<NinjaRuleVariable, NinjaVariableValue> ruleVariables;
- if (internedName.equals("phony")) {
- ruleVariables = ImmutableSortedMap.of();
- } else {
- NinjaRule ninjaRule = scope.findRule(offset, ruleName);
- if (ninjaRule == null) {
- throw new GenericParsingException(
- String.format("could not resolve rule '%s'", internedName));
- } else {
- ruleVariables =
- reduceRuleVariables(scope, offset, ninjaRule.getVariables(), variables, nameInterner);
- }
- }
- return new NinjaTarget(
- nameInterner.intern(ruleName),
- inputsBuilder.build(),
- outputsBuilder.build(),
- offset,
- ruleVariables);
- }
-
- /**
- * We expand the rule's variables with the following assumptions: Rule variables can refer to
- * target's variables (and file variables). Interdependence between rule variables can happen
- * only for 'command' variable, for now we ignore other possible dependencies between rule
- * variables (seems the only other variable which can meaningfully depend on sibling variables
- * is description, and currently we are ignoring it).
- *
- * <p>Also, for resolving rule's variables we are using scope+offset of target, according to
- * specification (https://ninja-build.org/manual.html#_variable_expansion).
- *
- * <p>See {@link NinjaRuleVariable} for the list.
- */
- private static ImmutableSortedMap<NinjaRuleVariable, NinjaVariableValue> reduceRuleVariables(
- NinjaScope targetScope,
- long targetOffset,
- Map<NinjaRuleVariable, NinjaVariableValue> ruleVariables,
- ImmutableSortedMap<String, String> targetVariables,
- Interner<String> interner) {
- ImmutableSortedMap.Builder<String, List<Pair<Long, String>>> variablesBuilder =
- ImmutableSortedMap.naturalOrder();
- targetVariables.forEach(
- (key, value) -> variablesBuilder.put(key, ImmutableList.of(Pair.of(0L, value))));
- NinjaScope scopeWithVariables =
- targetScope.createScopeFromExpandedValues(variablesBuilder.buildOrThrow());
-
- ImmutableSortedMap.Builder<NinjaRuleVariable, NinjaVariableValue> builder =
- ImmutableSortedMap.naturalOrder();
-
- // Description is taken from the "build" statement (instead of the referenced rule)
- // if it's available.
- boolean targetHasDescription = false;
- String targetVariable = targetVariables.get("description");
- if (targetVariable != null) {
- builder.put(
- NinjaRuleVariable.DESCRIPTION, NinjaVariableValue.createPlainText(targetVariable));
- targetHasDescription = true;
- }
-
- // symlink_outputs are taken from the "build" statement (instead of the referenced rule)
- // if it's available.
- boolean targetHasSymlinkOutputs = false;
- String symlinkOutputs = targetVariables.get("symlink_outputs");
- if (symlinkOutputs != null) {
- builder.put(
- NinjaRuleVariable.SYMLINK_OUTPUTS, NinjaVariableValue.createPlainText(symlinkOutputs));
- targetHasSymlinkOutputs = true;
- }
-
- for (Map.Entry<NinjaRuleVariable, NinjaVariableValue> entry : ruleVariables.entrySet()) {
- NinjaRuleVariable type = entry.getKey();
-
- // Don't use the rule variable if the same variable is defined in the build statement.
- if (type == NinjaRuleVariable.DESCRIPTION && targetHasDescription) {
- continue;
- }
- if (type == NinjaRuleVariable.SYMLINK_OUTPUTS && targetHasSymlinkOutputs) {
- continue;
- }
- NinjaVariableValue reducedValue =
- scopeWithVariables.getReducedValue(
- targetOffset, entry.getValue(), INPUTS_OUTPUTS_VARIABLES, interner);
- builder.put(type, reducedValue);
- }
- return builder.buildOrThrow();
- }
- }
-
- /** Enum with possible kinds of inputs. */
- @Immutable
- public enum InputKind implements InputOutputKind {
- EXPLICIT,
- IMPLICIT,
- ORDER_ONLY,
- VALIDATION,
- }
-
- /** Enum with possible kinds of outputs. */
- @Immutable
- public enum OutputKind implements InputOutputKind {
- EXPLICIT,
- IMPLICIT
- }
-
- /**
- * Marker interface, so that it is possible to address {@link InputKind} and {@link OutputKind}
- * together in one map.
- */
- @Immutable
- public interface InputOutputKind {}
-
- private final String ruleName;
- private final ImmutableSortedKeyListMultimap<InputKind, PathFragment> inputs;
- private final ImmutableSortedKeyListMultimap<OutputKind, PathFragment> outputs;
- private final long offset;
-
- /**
- * A "reduced" set of ninja rule variables. All variables are expanded except for those in {@link
- * #INPUTS_OUTPUTS_VARIABLES}, as this saves memory.
- */
- private final ImmutableSortedMap<NinjaRuleVariable, NinjaVariableValue> ruleVariables;
-
- private NinjaTarget(
- String ruleName,
- ImmutableSortedKeyListMultimap<InputKind, PathFragment> inputs,
- ImmutableSortedKeyListMultimap<OutputKind, PathFragment> outputs,
- long offset,
- ImmutableSortedMap<NinjaRuleVariable, NinjaVariableValue> ruleVariables) {
- this.ruleName = ruleName;
- this.inputs = inputs;
- this.outputs = outputs;
- this.offset = offset;
- this.ruleVariables = ruleVariables;
- }
-
- public String getRuleName() {
- return ruleName;
- }
-
- public List<PathFragment> getOutputs() {
- return outputs.get(OutputKind.EXPLICIT);
- }
-
- public List<PathFragment> getImplicitOutputs() {
- return outputs.get(OutputKind.IMPLICIT);
- }
-
- public Collection<PathFragment> getAllOutputs() {
- return outputs.values();
- }
-
- public Collection<PathFragment> getAllSymlinkOutputs() {
- String symlinkOutputs = computeRuleVariables().get(NinjaRuleVariable.SYMLINK_OUTPUTS);
- if (symlinkOutputs == null) {
- return ImmutableSet.of();
- } else {
- return Arrays.stream(symlinkOutputs.split(" "))
- .map(PathFragment::create)
- .collect(Collectors.toSet());
- }
- }
-
- public Collection<PathFragment> getAllInputs() {
- return inputs.values();
- }
-
- public Collection<Map.Entry<InputKind, PathFragment>> getAllInputsAndKind() {
- return inputs.entries();
- }
-
- public Collection<PathFragment> getExplicitInputs() {
- return inputs.get(InputKind.EXPLICIT);
- }
-
- public Collection<PathFragment> getImplicitInputs() {
- return inputs.get(InputKind.IMPLICIT);
- }
-
- public Collection<PathFragment> getOrderOnlyInputs() {
- return inputs.get(InputKind.ORDER_ONLY);
- }
-
- public Collection<PathFragment> getValidationInputs() {
- return inputs.get(InputKind.VALIDATION);
- }
-
- public long getOffset() {
- return offset;
- }
-
- public static Builder builder(NinjaScope scope, long offset, Interner<String> nameInterner) {
- return new Builder(scope, offset, nameInterner);
- }
-
- public String prettyPrint() {
- return "build "
- + prettyPrintPaths("\n", getOutputs())
- + prettyPrintPaths("\n| ", getImplicitOutputs())
- + "\n: "
- + this.ruleName
- + prettyPrintPaths("\n", getExplicitInputs())
- + prettyPrintPaths("\n| ", getImplicitInputs())
- + prettyPrintPaths("\n|| ", getOrderOnlyInputs());
- }
-
- @Override
- public String toString() {
- return prettyPrint();
- }
-
- @Override
- public int hashCode() {
- return Long.hashCode(offset);
- }
-
- /**
- * Returns a map from rule variable to fully-expanded value, for all rule variables defined in
- * this target.
- */
- public ImmutableSortedMap<NinjaRuleVariable, String> computeRuleVariables() {
- ImmutableSortedMap<String, String> lateExpansionVariables = computeInputOutputVariables();
- ImmutableSortedMap.Builder<String, String> fullExpansionVariablesBuilder =
- ImmutableSortedMap.<String, String>naturalOrder().putAll(lateExpansionVariables);
-
- ImmutableSortedMap.Builder<NinjaRuleVariable, String> builder =
- ImmutableSortedMap.naturalOrder();
- for (Map.Entry<NinjaRuleVariable, NinjaVariableValue> entry : ruleVariables.entrySet()) {
- NinjaRuleVariable type = entry.getKey();
- // Skip command for now. It may need to expand other rule variables.
- if (NinjaRuleVariable.COMMAND.equals(type)) {
- continue;
- }
-
- String expandedValue = entry.getValue().expandValue(lateExpansionVariables);
- builder.put(type, expandedValue);
- fullExpansionVariablesBuilder.put(Ascii.toLowerCase(type.name()), expandedValue);
- }
-
- // TODO(cparsons): Ensure parsing exception is thrown early if the rule has no command defined.
- // Otherwise, this throws NPE.
- String expandedCommand =
- ruleVariables
- .get(NinjaRuleVariable.COMMAND)
- .expandValue(fullExpansionVariablesBuilder.buildOrThrow());
- builder.put(NinjaRuleVariable.COMMAND, expandedCommand);
- return builder.buildOrThrow();
- }
-
- private ImmutableSortedMap<String, String> computeInputOutputVariables() {
- ImmutableSortedMap.Builder<String, String> builder = ImmutableSortedMap.naturalOrder();
- String inNewline =
- inputs.get(InputKind.EXPLICIT).stream()
- .map(PathFragment::getPathString)
- .collect(Collectors.joining("\n"));
- String out =
- outputs.get(OutputKind.EXPLICIT).stream()
- .map(PathFragment::getPathString)
- .collect(Collectors.joining(" "));
- builder.put("in", inNewline.replace('\n', ' '));
- builder.put("in_newline", inNewline);
- builder.put("out", out);
- return builder.buildOrThrow();
- }
-
- private static String prettyPrintPaths(String startDelimiter, Collection<PathFragment> paths) {
- if (paths.isEmpty()) {
- return "";
- }
- return startDelimiter
- + paths.stream().map(PathFragment::getPathString).collect(Collectors.joining("$\n"));
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaVariableValue.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaVariableValue.java
deleted file mode 100644
index a5c0520..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser/NinjaVariableValue.java
+++ /dev/null
@@ -1,137 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja.parser;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-/**
- * Ninja variable value.
- *
- * <p>Can contain references to the other variables, defined earlier in the scope (or parent scope).
- * It is expected that those references can be replaced in one step, as all the variables are
- * parsed, so this particular structure is only needed to keep the intermediate state.
- */
-public class NinjaVariableValue {
- /**
- * List of functions for computing parts of the variable value. For the case of text part, the
- * function just returns the text literal. For the case of variable reference, function calls the
- * value expander, passed to it as an argument, to get the variable value, and returns it.
- *
- * <p>{@link NinjaVariableValue.Builder#addVariable(String)}
- */
- private final ImmutableList<Function<Function<String, String>, String>> parts;
- /** Indicates that this value contain no variable references, i.e. contain only plain text. */
- private final boolean isPlainText;
-
- private NinjaVariableValue(
- ImmutableList<Function<Function<String, String>, String>> parts, boolean isPlainText) {
- this.parts = parts;
- this.isPlainText = isPlainText;
- }
-
- /** Created the value wrapping some plain text. */
- public static NinjaVariableValue createPlainText(String text) {
- return builder().addText(text).build();
- }
-
- public boolean isPlainText() {
- return isPlainText;
- }
-
- /** Compute the expanded value, using the passed <code>expander</code> function. */
- @VisibleForTesting
- public String getExpandedValue(Function<String, String> expander) {
- return parts.stream().map(fun -> fun.apply(expander)).collect(Collectors.joining(""));
- }
-
- /**
- * Returns a "reduced" representation of this value by expanding all variables for which an
- * expanded value is available. If the given expansion function returns null for any part of the
- * value, that part will remain unexpanded in the returned value.
- */
- public NinjaVariableValue reduce(Function<String, String> expander) {
- NinjaVariableValue.Builder builder = builder();
- for (Function<Function<String, String>, String> part : parts) {
- String result = part.apply(expander);
- if (result != null) {
- builder.addText(result);
- } else {
- builder.addPart(part);
- }
- }
- return builder.build();
- }
-
- /**
- * Returns the fully expanded string that this value represents, given a map from variable name to
- * expanded value.
- *
- * <p>If this value contains an unexpanded variable for which no value is present in the given
- * map, the empty string is used as the value of that variable.
- */
- public String expandValue(ImmutableMap<String, String> map) {
- return getExpandedValue(s -> Strings.nullToEmpty(map.get(s)));
- }
-
- /**
- * Compute the presentation of this value, replacing the variable references with ${reference}.
- */
- public String getRawText() {
- return getExpandedValue(s -> String.format("${%s}", s));
- }
-
- public static Builder builder() {
- return new Builder();
- }
-
- /** Builder class for {@link NinjaVariableValue}. */
- public static final class Builder {
- private final ImmutableList.Builder<Function<Function<String, String>, String>> builder;
- private boolean isPrimitive = true;
-
- private Builder() {
- this.builder = ImmutableList.builder();
- }
-
- /** Add plain text fragment. */
- public Builder addText(String text) {
- builder.add(expander -> text);
- return this;
- }
-
- /** Add reference to variable <code>name</code>. */
- public Builder addVariable(String name) {
- builder.add(expander -> expander.apply(name));
- isPrimitive = false;
- return this;
- }
-
- private Builder addPart(Function<Function<String, String>, String> part) {
- builder.add(part);
- isPrimitive = false;
- return this;
- }
-
- public NinjaVariableValue build() {
- return new NinjaVariableValue(builder.build(), isPrimitive);
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline/BUILD
deleted file mode 100644
index f0b7344..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline/BUILD
+++ /dev/null
@@ -1,39 +0,0 @@
-load("@rules_java//java:defs.bzl", "java_library")
-
-package(
- default_visibility = ["//src:__subpackages__"],
-)
-
-filegroup(
- name = "srcs",
- srcs = glob(["*"]),
- visibility = ["//src:__subpackages__"],
-)
-
-java_library(
- name = "pipeline",
- srcs = ["NinjaPipeline.java"],
- deps = [
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser:parser_impl",
- "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
- "//third_party:guava",
- ],
-)
-
-java_library(
- name = "pipeline_impl",
- srcs = ["NinjaPipelineImpl.java"],
- deps = [
- ":pipeline",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser:parser_impl",
- "//src/main/java/com/google/devtools/build/lib/concurrent",
- "//src/main/java/com/google/devtools/build/lib/util",
- "//src/main/java/com/google/devtools/build/lib/vfs",
- "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
- "//third_party:guava",
- ],
-)
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline/NinjaPipeline.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline/NinjaPipeline.java
deleted file mode 100644
index b87812d..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline/NinjaPipeline.java
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.pipeline;
-
-import com.google.common.collect.Interner;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaFileParseResult;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaFileParseResult.NinjaPromise;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaScope;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaVariableValue;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.io.IOException;
-
-/**
- * Interface Responsible for parsing Ninja file, all its included and subninja files, and returning
- * {@link NinjaScope} with rules and expanded variables, and list of {@link NinjaTarget}.
- *
- * <p>This interface exists to break the package cycle between parser and pipeline.
- */
-public interface NinjaPipeline {
-
- /**
- * Synchronously or asynchronously schedules parsing of the included Ninja file, and returns
- * {@link NinjaPromise<NinjaFileParseResult>} - an object, from which we can obtain the result of
- * the parsing - {@link NinjaFileParseResult} - after we resolved variables in the parent file to
- * the point when the child file was included.
- *
- * <p>In some cases, include and subninja statements can contain variable references, then we must
- * postpone scheduling the parsing of the file until variable is expanded and the path to file
- * becomes known.
- *
- * <p>That is why we use {@link NinjaPromise<NinjaFileParseResult>} to save the future file
- * parsing result in the parent file {@link NinjaFileParseResult} structure.
- */
- NinjaPromise<NinjaFileParseResult> createChildFileParsingPromise(
- NinjaVariableValue value, long offset, String parentNinjaFileName)
- throws GenericParsingException, IOException;
-
- /**
- * An interner for {@link PathFragment} instances for the inputs and outputs of {@link
- * NinjaTarget}.
- */
- Interner<PathFragment> getPathFragmentInterner();
-
- /** An String interner for rule and build statements' variable names. */
- Interner<String> getNameInterner();
-}
-
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline/NinjaPipelineImpl.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline/NinjaPipelineImpl.java
deleted file mode 100644
index 0d4d206..0000000
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline/NinjaPipelineImpl.java
+++ /dev/null
@@ -1,227 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja.pipeline;
-
-import static com.google.devtools.build.lib.concurrent.MoreFutures.waitForFutureAndGetWithCheckedException;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Interner;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.CollectingListFuture;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ParallelFileProcessing;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ParallelFileProcessing.BlockParameters;
-import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaFileParseResult;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaFileParseResult.NinjaPromise;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaParser;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaParserStep;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaScope;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaVariableValue;
-import com.google.devtools.build.lib.concurrent.BlazeInterners;
-import com.google.devtools.build.lib.util.Pair;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.io.IOException;
-import java.nio.channels.ReadableByteChannel;
-import java.util.ArrayDeque;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Responsible for parsing Ninja file, all its included and subninja files, and returning {@link
- * NinjaScope} with rules and expanded variables, and list of {@link NinjaTarget}.
- *
- * <p>Uses provided {@link ListeningExecutorService} for scheduling tasks in parallel.
- */
-public class NinjaPipelineImpl implements NinjaPipeline {
- private final Path basePath;
- private final ListeningExecutorService service;
- private final Collection<Path> includedOrSubninjaFiles;
- private final String ownerTargetName;
- private final Set<Path> childPaths;
- private Integer readBlockSize;
-
- private final Interner<PathFragment> pathFragmentInterner = BlazeInterners.newWeakInterner();
- private final Interner<String> nameInterner = BlazeInterners.newWeakInterner();
-
- /**
- * @param basePath base path for resolving include and subninja paths.
- * @param service service to use for scheduling tasks in parallel.
- * @param includedOrSubninjaFiles Ninja files expected in include/subninja statements
- * @param ownerTargetName name of the owner ninja_graph target
- */
- public NinjaPipelineImpl(
- Path basePath,
- ListeningExecutorService service,
- Collection<Path> includedOrSubninjaFiles,
- String ownerTargetName) {
- this.basePath = basePath;
- this.service = service;
- this.includedOrSubninjaFiles = includedOrSubninjaFiles;
- this.ownerTargetName = ownerTargetName;
- this.childPaths = Sets.newConcurrentHashSet();
- }
-
- /**
- * Parses <code>mainFile</code> and all it's children from include and subninja statements.
- *
- * @return {@link Pair} of {@link NinjaScope} with rules and expanded variables (and child
- * scopes), and list of {@link NinjaTarget}.
- */
- public List<NinjaTarget> pipeline(Path mainFile)
- throws GenericParsingException, InterruptedException, IOException {
- NinjaFileParseResult result =
- waitForFutureAndGetWithCheckedException(
- scheduleParsing(mainFile), GenericParsingException.class, IOException.class);
-
- Map<NinjaScope, List<FileFragment>> rawTargets = Maps.newHashMap();
- NinjaScope scope = new NinjaScope();
- // This will cause additional parsing of included/subninja scopes, and their recursive expand.
- result.expandIntoScope(scope, rawTargets);
- return iterateScopesScheduleTargetsParsing(scope, rawTargets);
- }
-
- /**
- * Each NinjaTarget should be parsed in the context of it's parent {@link NinjaScope}. (All the
- * variables in targets are immediately expanded.) We are iterating main and all transitively
- * included scopes, and parsing corresponding targets.
- */
- private List<NinjaTarget> iterateScopesScheduleTargetsParsing(
- NinjaScope scope, Map<NinjaScope, List<FileFragment>> rawTargets)
- throws GenericParsingException, InterruptedException {
- ArrayDeque<NinjaScope> queue = new ArrayDeque<>();
- queue.add(scope);
- CollectingListFuture<NinjaTarget, GenericParsingException> future =
- new CollectingListFuture<>(GenericParsingException.class);
- while (!queue.isEmpty()) {
- NinjaScope currentScope = queue.removeFirst();
- List<FileFragment> targetFragments = rawTargets.get(currentScope);
- Preconditions.checkNotNull(targetFragments);
- for (FileFragment fragment : targetFragments) {
- future.add(
- service.submit(
- () ->
- new NinjaParserStep(
- new NinjaLexer(fragment), pathFragmentInterner, nameInterner)
- .parseNinjaTarget(currentScope, fragment.getFragmentOffset())));
- }
- queue.addAll(currentScope.getIncludedScopes());
- queue.addAll(currentScope.getSubNinjaScopes());
- }
- return future.getResult();
- }
-
- @Override
- public NinjaPromise<NinjaFileParseResult> createChildFileParsingPromise(
- NinjaVariableValue value, long offset, String parentNinjaFileName)
- throws GenericParsingException, IOException {
- if (value.isPlainText()) {
- // If the value of the path is already known, we can immediately schedule parsing
- // of the child Ninja file.
- Path path = getChildNinjaPath(value.getRawText(), parentNinjaFileName);
- ListenableFuture<NinjaFileParseResult> parsingFuture = scheduleParsing(path);
- return (scope) ->
- waitForFutureAndGetWithCheckedException(
- parsingFuture, GenericParsingException.class, IOException.class);
- } else {
- // If the value of the child path refers some variables in the parent scope, resolve it,
- // when the lambda is called, schedule the parsing and wait for it's completion.
- return (scope) -> {
- String expandedValue = scope.getExpandedValue(offset, value);
- if (expandedValue.isEmpty()) {
- throw new GenericParsingException("Expected non-empty path.");
- }
- Path path = getChildNinjaPath(expandedValue, parentNinjaFileName);
- return waitForFutureAndGetWithCheckedException(
- scheduleParsing(path), GenericParsingException.class, IOException.class);
- };
- }
- }
-
- /**
- * Set the size of the block read by {@link ParallelFileProcessing}. Method is mainly intended to
- * be used in tests.
- */
- @VisibleForTesting
- public void setReadBlockSize(Integer readBlockSize) {
- this.readBlockSize = readBlockSize;
- }
-
- private Path getChildNinjaPath(String rawText, String parentNinjaFileName)
- throws GenericParsingException {
- Path childPath = basePath.getRelative(rawText);
- if (!this.includedOrSubninjaFiles.contains(childPath)) {
- throw new GenericParsingException(
- String.format(
- "Ninja file '%s' requested from '%s' not declared in 'ninja_srcs' attribute of '%s'.",
- rawText, parentNinjaFileName, this.ownerTargetName));
- }
- return childPath;
- }
-
- /**
- * Actually schedules the parsing of the Ninja file and returns {@link
- * ListenableFuture<NinjaFileParseResult>} for obtaining the result.
- */
- private ListenableFuture<NinjaFileParseResult> scheduleParsing(Path path)
- throws IOException, GenericParsingException {
- if (!this.childPaths.add(path)) {
- throw new GenericParsingException(
- String.format(
- "Detected cycle or duplicate inclusion in Ninja files dependencies, including '%s'.",
- path.getBaseName()));
- }
- BlockParameters parameters = new BlockParameters(path.getFileSize());
- if (readBlockSize != null) {
- parameters.setReadBlockSize(readBlockSize);
- }
- return service.submit(
- () -> {
- try (ReadableByteChannel channel = path.createReadableByteChannel()) {
- List<NinjaFileParseResult> pieces = Lists.newArrayList();
- ParallelFileProcessing.processFile(
- channel,
- parameters,
- () -> {
- NinjaFileParseResult parseResult = new NinjaFileParseResult();
- pieces.add(parseResult);
- return new NinjaParser(this, parseResult, path.getBaseName());
- },
- service);
- return NinjaFileParseResult.merge(pieces);
- }
- });
- }
-
- @Override
- public Interner<PathFragment> getPathFragmentInterner() {
- return pathFragmentInterner;
- }
-
- @Override
- public Interner<String> getNameInterner() {
- return nameInterner;
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java
index b9cc320..1bbdcf4 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java
@@ -206,11 +206,7 @@
Set<AspectKey> builtAspects = new HashSet<>();
try (SilentCloseable c = Profiler.instance().profile("ExecutionTool.init")) {
- executionTool.prepareForExecution(
- request.getId(),
- builtTargets,
- builtAspects,
- loadingResult.getNotSymlinkedInExecrootDirectories());
+ executionTool.prepareForExecution(request.getId(), builtTargets, builtAspects);
}
// TODO(b/199053098): implement support for --nobuild.
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
index 3022626..f4d0de9 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
@@ -23,7 +23,6 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.flogger.GoogleLogger;
import com.google.devtools.build.lib.actions.Action;
@@ -247,10 +246,7 @@
* tests for these setup steps.
*/
public void prepareForExecution(
- UUID buildId,
- Set<ConfiguredTargetKey> builtTargets,
- Set<AspectKey> builtAspects,
- ImmutableSortedSet<String> notSymlinkedInExecrootDirectories)
+ UUID buildId, Set<ConfiguredTargetKey> builtTargets, Set<AspectKey> builtAspects)
throws AbruptExitException, BuildFailedException, InterruptedException {
init();
BuildRequestOptions options = request.getBuildOptions();
@@ -272,7 +268,6 @@
getExecRoot(),
singleSourceRoot.asPath(),
/*prefix=*/ env.getDirectories().getProductName() + "-",
- notSymlinkedInExecrootDirectories,
request.getOptions(BuildLanguageOptions.class).experimentalSiblingRepositoryLayout);
} catch (IOException e) {
throw new AbruptExitException(
@@ -382,7 +377,7 @@
boolean useEventBasedBuildCompletionStatus)
throws BuildFailedException, InterruptedException, TestExecException, AbruptExitException {
Stopwatch timer = Stopwatch.createStarted();
- prepare(packageRoots, analysisResult.getNonSymlinkedDirectoriesUnderExecRoot());
+ prepare(packageRoots);
ActionGraph actionGraph = analysisResult.getActionGraph();
@@ -580,9 +575,7 @@
}
}
- private void prepare(
- PackageRoots packageRoots, ImmutableSortedSet<String> nonSymlinkedDirectoriesUnderExecRoot)
- throws AbruptExitException, InterruptedException {
+ private void prepare(PackageRoots packageRoots) throws AbruptExitException, InterruptedException {
Optional<ImmutableMap<PackageIdentifier, Root>> packageRootMap =
packageRoots.getPackageRootsMap();
if (packageRootMap.isPresent()) {
@@ -596,7 +589,6 @@
packageRootMap.get(),
getExecRoot(),
runtime.getProductName(),
- nonSymlinkedDirectoriesUnderExecRoot,
request.getOptions(BuildLanguageOptions.class).experimentalSiblingRepositoryLayout);
symlinkForest.plantSymlinkForest();
} catch (IOException e) {
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/SymlinkForest.java b/src/main/java/com/google/devtools/build/lib/buildtool/SymlinkForest.java
index 5b948dd..324f9ff 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/SymlinkForest.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/SymlinkForest.java
@@ -17,7 +17,6 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -26,11 +25,7 @@
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.concurrent.ThreadSafety;
-import com.google.devtools.build.lib.server.FailureDetails;
-import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
-import com.google.devtools.build.lib.server.FailureDetails.SymlinkForest.Code;
import com.google.devtools.build.lib.util.AbruptExitException;
-import com.google.devtools.build.lib.util.DetailedExitCode;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
@@ -39,9 +34,6 @@
import java.util.Collections;
import java.util.Map;
import java.util.Set;
-import net.starlark.java.eval.Sequence;
-import net.starlark.java.eval.StarlarkThread;
-import net.starlark.java.syntax.Location;
/** Creates a symlink forest based on a package path map. */
public class SymlinkForest {
@@ -51,13 +43,12 @@
private final Path execroot;
private final String productName;
private final String prefix;
- private final ImmutableSortedSet<String> notSymlinkedInExecrootDirectories;
private final boolean siblingRepositoryLayout;
/** Constructor for a symlink forest creator without non-symlinked directories parameter. */
public SymlinkForest(
ImmutableMap<PackageIdentifier, Root> packageRoots, Path execroot, String productName) {
- this(packageRoots, execroot, productName, ImmutableSortedSet.of(), false);
+ this(packageRoots, execroot, productName, false);
}
/**
@@ -68,21 +59,16 @@
* @param packageRoots source package roots to which to create symlinks
* @param execroot path where to plant the symlink forest
* @param productName {@code BlazeRuntime#getProductName()}
- * @param notSymlinkedInExecrootDirectories directories to not symlink in exec root. {@link
- * com.google.devtools.build.lib.packages.WorkspaceGlobals#dontSymlinkDirectoriesInExecroot(Sequence,
- * Location, StarlarkThread)}
*/
public SymlinkForest(
ImmutableMap<PackageIdentifier, Root> packageRoots,
Path execroot,
String productName,
- ImmutableSortedSet<String> notSymlinkedInExecrootDirectories,
boolean siblingRepositoryLayout) {
this.packageRoots = packageRoots;
this.execroot = execroot;
this.productName = productName;
this.prefix = productName + "-";
- this.notSymlinkedInExecrootDirectories = notSymlinkedInExecrootDirectories;
this.siblingRepositoryLayout = siblingRepositoryLayout;
}
@@ -112,25 +98,12 @@
*/
@VisibleForTesting
@ThreadSafety.ThreadSafe
- static void deleteTreesBelowNotPrefixed(
- Path dir, String prefix, ImmutableSortedSet<String> notSymlinkedInExecrootDirectories)
- throws IOException {
-
+ static void deleteTreesBelowNotPrefixed(Path dir, String prefix) throws IOException {
for (Path p : dir.getDirectoryEntries()) {
-
if (p.getBaseName().startsWith(prefix)) {
continue;
}
- // If the path in question is a toplevel output directory, then it should not be deleted
- // from the execroot here because it was not created as part of symlink forest creation,
- // unless it is a symlink. If the path in question is a toplevel output directory and it is
- // a symlink, then this means that it was created as part of a previous build where it was
- // not a toplevel output directory at the time, and should be deleted.
- if (notSymlinkedInExecrootDirectories.contains(p.getBaseName()) && !p.isSymbolicLink()) {
- continue;
- }
-
p.deleteTree();
}
}
@@ -170,9 +143,6 @@
}
for (Path target : mainRepoRoot.getDirectoryEntries()) {
String baseName = target.getBaseName();
- if (this.notSymlinkedInExecrootDirectories.contains(baseName)) {
- continue;
- }
Path execPath = execroot.getRelative(baseName);
// Create any links that don't start with bazel-, and ignore external/ directory if
// user has it in the source tree because it conflicts with external repository location.
@@ -195,13 +165,6 @@
for (Map.Entry<Path, Path> entry : mainRepoLinks.entrySet()) {
Path link = entry.getKey();
Path target = entry.getValue();
- if (this.notSymlinkedInExecrootDirectories.contains(target.getBaseName())) {
- throw new AbruptExitException(
- detailedSymlinkForestExitCode(
- "Directories specified with toplevel_output_directories should be ignored and can"
- + " not be used as sources.",
- Code.TOPLEVEL_OUTDIR_USED_AS_SOURCE));
- }
link.createSymbolicLink(target);
plantedSymlinks.add(link);
}
@@ -339,7 +302,7 @@
* @return the symlinks that have been planted
*/
public ImmutableList<Path> plantSymlinkForest() throws IOException, AbruptExitException {
- deleteTreesBelowNotPrefixed(execroot, prefix, notSymlinkedInExecrootDirectories);
+ deleteTreesBelowNotPrefixed(execroot, prefix);
if (siblingRepositoryLayout) {
// Delete execroot/../<symlinks> to directories representing external repositories.
@@ -361,8 +324,7 @@
PackageIdentifier pkgId = entry.getKey();
if (pkgId.equals(LabelConstants.EXTERNAL_PACKAGE_IDENTIFIER)) {
// //external is a virtual package regardless , don't add it to the symlink tree.
- // Subpackages of
- // external, like //external/foo, are fine though.
+ // Subpackages of //external, like //external/foo, are fine though.
continue;
}
RepositoryName repository = pkgId.getRepository();
@@ -403,12 +365,6 @@
// removed in the future, we should remove the plantSymlinkForestMultiPackagePath
// implementation when --package_path is gone.
if (mainRepoRoots.size() > 1) {
- if (!this.notSymlinkedInExecrootDirectories.isEmpty()) {
- throw new AbruptExitException(
- detailedSymlinkForestExitCode(
- "toplevel_output_directories is not supported together with --package_path option.",
- Code.TOPLEVEL_OUTDIR_PACKAGE_PATH_CONFLICT));
- }
plantSymlinkForestMultiPackagePath(plantedSymlinks, packageRootsForMainRepo);
} else if (shouldLinkAllTopLevelItems) {
Path mainRepoRoot = Iterables.getOnlyElement(mainRepoRoots).asPath();
@@ -430,18 +386,13 @@
Path execroot,
Path sourceRoot,
String prefix,
- ImmutableSortedSet<String> notSymlinkedInExecrootDirectories,
boolean siblingRepositoryLayout)
throws IOException {
- deleteTreesBelowNotPrefixed(execroot, prefix, notSymlinkedInExecrootDirectories);
+ deleteTreesBelowNotPrefixed(execroot, prefix);
// Plant everything under the single source root.
for (Path target : sourceRoot.getDirectoryEntries()) {
-
String baseName = target.getBaseName();
- if (notSymlinkedInExecrootDirectories.contains(baseName)) {
- continue;
- }
Path execPath = execroot.getRelative(baseName);
// Create any links that don't start with bazel-, and ignore external/ directory if
// user has it in the source tree because it conflicts with external repository location.
@@ -453,14 +404,6 @@
}
}
- private static DetailedExitCode detailedSymlinkForestExitCode(String message, Code code) {
- return DetailedExitCode.of(
- FailureDetail.newBuilder()
- .setMessage(message)
- .setSymlinkForest(FailureDetails.SymlinkForest.newBuilder().setCode(code))
- .build());
- }
-
private static PackageIdentifier createInRepo(
PackageIdentifier repo, PathFragment packageFragment) {
return PackageIdentifier.create(repo.getRepository(), packageFragment);
diff --git a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactory.java b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactory.java
index f5ad9a5..aa015c5 100644
--- a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactory.java
@@ -16,7 +16,6 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSortedSet;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.events.Event;
@@ -404,8 +403,4 @@
public Map<PathFragment, RepositoryName> getManagedDirectories() {
return workspaceGlobals.getManagedDirectories();
}
-
- public ImmutableSortedSet<String> getDoNotSymlinkInExecrootPaths() {
- return workspaceGlobals.getDoNotSymlinkInExecrootPaths();
- }
}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFileValue.java b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFileValue.java
index ebfaa96..fa1061a 100644
--- a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFileValue.java
+++ b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFileValue.java
@@ -16,7 +16,6 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Interner;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.concurrent.BlazeInterners;
@@ -108,8 +107,6 @@
// Mapping of the relative paths of the incrementally updated managed directories
// to the managing external repositories
private final ImmutableMap<PathFragment, RepositoryName> managedDirectories;
- // Directories to be excluded from symlinking to the execroot.
- private final ImmutableSortedSet<String> doNotSymlinkInExecrootPaths;
/**
* Create a WorkspaceFileValue containing the various values necessary to compute the split
@@ -128,7 +125,6 @@
* second one and so on).
* @param hasNext Is there a next part in the WORKSPACE file or this part the last one?
* @param managedDirectories Mapping of the relative paths of the incrementally updated managed
- * @param doNotSymlinkInExecrootPaths directories to be excluded from symlinking to the execroot
*/
public WorkspaceFileValue(
Package pkg,
@@ -138,8 +134,7 @@
RootedPath path,
int idx,
boolean hasNext,
- ImmutableMap<PathFragment, RepositoryName> managedDirectories,
- ImmutableSortedSet<String> doNotSymlinkInExecrootPaths) {
+ ImmutableMap<PathFragment, RepositoryName> managedDirectories) {
this.pkg = Preconditions.checkNotNull(pkg);
this.idx = idx;
this.path = path;
@@ -149,7 +144,6 @@
this.loadToChunkMap = ImmutableMap.copyOf(loadToChunkMap);
this.repositoryMapping = pkg.getExternalPackageRepositoryMappings();
this.managedDirectories = managedDirectories;
- this.doNotSymlinkInExecrootPaths = doNotSymlinkInExecrootPaths;
}
/**
@@ -235,8 +229,4 @@
public ImmutableMap<PathFragment, RepositoryName> getManagedDirectories() {
return managedDirectories;
}
-
- public ImmutableSortedSet<String> getDoNotSymlinkInExecrootPaths() {
- return doNotSymlinkInExecrootPaths;
- }
}
diff --git a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceGlobals.java b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceGlobals.java
index 246372f..dd261b0 100644
--- a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceGlobals.java
+++ b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceGlobals.java
@@ -18,10 +18,8 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
import com.google.devtools.build.lib.cmdline.BazelModuleContext;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
@@ -36,7 +34,6 @@
import com.google.devtools.build.lib.vfs.PathFragment;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Pattern;
import javax.annotation.Nullable;
@@ -58,8 +55,6 @@
// Mapping of the relative paths of the incrementally updated managed directories
// to the managing external repositories
private final TreeMap<PathFragment, RepositoryName> managedDirectoriesMap;
- // Directories to be excluded from symlinking to the execroot.
- private ImmutableSortedSet<String> doNotSymlinkInExecrootPaths;
public WorkspaceGlobals(boolean allowOverride, RuleFactory ruleFactory) {
this.allowOverride = allowOverride;
@@ -110,38 +105,6 @@
Dict.cast(managedDirectories, String.class, Object.class, "managed_directories"));
}
- @Override
- public void dontSymlinkDirectoriesInExecroot(Sequence<?> paths, StarlarkThread thread)
- throws EvalException {
- List<String> pathsList = Sequence.cast(paths, String.class, "paths");
- Set<String> set = Sets.newHashSet();
- for (String path : pathsList) {
- PathFragment pathFragment = PathFragment.create(path);
- if (pathFragment.isEmpty()) {
- throw Starlark.errorf("Empty path can not be passed to toplevel_output_directories.");
- }
- if (pathFragment.containsUplevelReferences() || pathFragment.isMultiSegment()) {
- throw Starlark.errorf(
- "toplevel_output_directories can only accept top level directories under"
- + " workspace, \"%s\" can not be specified as an attribute.",
- path);
- }
- if (pathFragment.isAbsolute()) {
- throw Starlark.errorf(
- "toplevel_output_directories can only accept top level directories under"
- + " workspace, absolute path \"%s\" can not be specified as an attribute.",
- path);
- }
- if (!set.add(pathFragment.getBaseName())) {
- throw Starlark.errorf(
- "toplevel_output_directories should not contain duplicate values: \"%s\" is"
- + " specified more then once.",
- path);
- }
- }
- doNotSymlinkInExecrootPaths = ImmutableSortedSet.copyOf(set);
- }
-
private void parseManagedDirectories(
Map<String, ?> managedDirectories) // <String, Sequence<String>>
throws EvalException {
@@ -220,10 +183,6 @@
return managedDirectoriesMap;
}
- public ImmutableSortedSet<String> getDoNotSymlinkInExecrootPaths() {
- return doNotSymlinkInExecrootPaths;
- }
-
private static RepositoryName getRepositoryName(@Nullable Label label) {
if (label == null) {
// registration happened directly in the main WORKSPACE
diff --git a/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java b/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java
index 5636dca..f2b3220 100644
--- a/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java
@@ -168,15 +168,6 @@
public boolean experimentalGoogleLegacyApi;
@Option(
- name = "experimental_ninja_actions",
- defaultValue = "false",
- documentationCategory = OptionDocumentationCategory.STARLARK_SEMANTICS,
- effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION},
- metadataTags = {OptionMetadataTag.EXPERIMENTAL},
- help = "If set to true, enables Ninja execution functionality.")
- public boolean experimentalNinjaActions;
-
- @Option(
name = "experimental_platforms_api",
defaultValue = "false",
documentationCategory = OptionDocumentationCategory.STARLARK_SEMANTICS,
@@ -567,7 +558,6 @@
INCOMPATIBLE_EXISTING_RULES_IMMUTABLE_VIEW, incompatibleExistingRulesImmutableView)
.setBool(EXPERIMENTAL_ACTION_RESOURCE_SET, experimentalActionResourceSet)
.setBool(EXPERIMENTAL_GOOGLE_LEGACY_API, experimentalGoogleLegacyApi)
- .setBool(EXPERIMENTAL_NINJA_ACTIONS, experimentalNinjaActions)
.setBool(EXPERIMENTAL_PLATFORMS_API, experimentalPlatformsApi)
.setBool(EXPERIMENTAL_CC_SHARED_LIBRARY, experimentalCcSharedLibrary)
.setBool(EXPERIMENTAL_REPO_REMOTE_EXEC, experimentalRepoRemoteExec)
@@ -637,7 +627,6 @@
public static final String INCOMPATIBLE_EXISTING_RULES_IMMUTABLE_VIEW =
"+incompatible_existing_rules_immutable_view";
public static final String EXPERIMENTAL_GOOGLE_LEGACY_API = "-experimental_google_legacy_api";
- public static final String EXPERIMENTAL_NINJA_ACTIONS = "-experimental_ninja_actions";
public static final String EXPERIMENTAL_PLATFORMS_API = "-experimental_platforms_api";
public static final String EXPERIMENTAL_REPO_REMOTE_EXEC = "-experimental_repo_remote_exec";
public static final String EXPERIMENTAL_SIBLING_REPOSITORY_LAYOUT =
diff --git a/src/main/java/com/google/devtools/build/lib/repository/ExternalPackageHelper.java b/src/main/java/com/google/devtools/build/lib/repository/ExternalPackageHelper.java
index fe8a8dd..0e8521d 100644
--- a/src/main/java/com/google/devtools/build/lib/repository/ExternalPackageHelper.java
+++ b/src/main/java/com/google/devtools/build/lib/repository/ExternalPackageHelper.java
@@ -16,7 +16,6 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.FileValue;
import com.google.devtools.build.lib.cmdline.LabelConstants;
@@ -44,30 +43,6 @@
this.workspaceFilesByPriority = workspaceFilesByPriority;
}
- /**
- * Returns directories, that should not be symlinked under the execroot.
- *
- * <p>Searches for toplevel_output_directories calls in the WORKSPACE file, and gathers values of
- * all "paths" attributes.
- */
- public ImmutableSortedSet<String> getNotSymlinkedInExecrootDirectories(Environment env)
- throws InterruptedException {
- ImmutableSortedSet.Builder<String> builder = ImmutableSortedSet.naturalOrder();
- WorkspaceFileValueProcessor gatherer =
- workspaceFileValue -> {
- ImmutableSortedSet<String> paths = workspaceFileValue.getDoNotSymlinkInExecrootPaths();
- if (paths != null) {
- builder.addAll(paths);
- }
- // Continue to read all the fragments.
- return true;
- };
- if (!iterateWorkspaceFragments(env, gatherer)) {
- return null;
- }
- return builder.build();
- }
-
/** Uses a rule name to fetch the corresponding Rule from the external package. */
@Nullable
public Rule getRuleByName(String ruleName, Environment env)
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java
index 7f2b07f..b0b2e56 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java
@@ -110,8 +110,7 @@
throws ArtifactFunctionException, InterruptedException {
Artifact artifact = (Artifact) skyKey;
if (!artifact.hasKnownGeneratingAction()) {
- // If the artifact has no known generating action, it is either a source artifact, or a
- // NinjaMysteryArtifact, which undergoes the same handling here.
+ // If the artifact has no known generating action, it is a source artifact.
return createSourceValue(artifact, env);
}
Artifact.DerivedArtifact derivedArtifact = (DerivedArtifact) artifact;
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
index 02744aa..eec580a 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -540,8 +540,7 @@
map.put(SkyFunctions.IGNORED_PACKAGE_PREFIXES, ignoredPackagePrefixesFunction);
map.put(SkyFunctions.TESTS_IN_SUITE, new TestExpansionFunction());
map.put(SkyFunctions.TEST_SUITE_EXPANSION, new TestsForTargetPatternFunction());
- map.put(
- SkyFunctions.TARGET_PATTERN_PHASE, new TargetPatternPhaseFunction(externalPackageHelper));
+ map.put(SkyFunctions.TARGET_PATTERN_PHASE, new TargetPatternPhaseFunction());
map.put(
SkyFunctions.PREPARE_ANALYSIS_PHASE, new PrepareAnalysisPhaseFunction(ruleClassProvider));
map.put(SkyFunctions.RECURSIVE_PKG, new RecursivePkgFunction(directories));
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternPhaseFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternPhaseFunction.java
index 9e68aa1..a4a81fc 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternPhaseFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternPhaseFunction.java
@@ -22,7 +22,6 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSetMultimap;
-import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.devtools.build.lib.bugreport.BugReport;
@@ -48,7 +47,6 @@
import com.google.devtools.build.lib.pkgcache.ParsingFailedEvent;
import com.google.devtools.build.lib.pkgcache.TargetParsingCompleteEvent;
import com.google.devtools.build.lib.pkgcache.TestFilter;
-import com.google.devtools.build.lib.repository.ExternalPackageHelper;
import com.google.devtools.build.lib.skyframe.TargetPatternPhaseValue.TargetPatternPhaseKey;
import com.google.devtools.build.lib.skyframe.TargetPatternValue.TargetPatternKey;
import com.google.devtools.build.lib.vfs.PathFragment;
@@ -70,23 +68,12 @@
* resolved Targets.
*/
final class TargetPatternPhaseFunction implements SkyFunction {
-
- private final ExternalPackageHelper externalPackageHelper;
-
- public TargetPatternPhaseFunction(ExternalPackageHelper externalPackageHelper) {
- this.externalPackageHelper = externalPackageHelper;
- }
+ public TargetPatternPhaseFunction() {}
@Override
public TargetPatternPhaseValue compute(SkyKey key, Environment env) throws InterruptedException {
TargetPatternPhaseKey options = (TargetPatternPhaseKey) key.argument();
WorkspaceNameValue workspaceName = (WorkspaceNameValue) env.getValue(WorkspaceNameValue.key());
- ImmutableSortedSet<String> notSymlinkedInExecrootDirectories =
- externalPackageHelper.getNotSymlinkedInExecrootDirectories(env);
- if (env.valuesMissing()) {
- return null;
- }
-
RepositoryMappingValue repositoryMappingValue =
(RepositoryMappingValue) env.getValue(RepositoryMappingValue.key(RepositoryName.MAIN));
if (repositoryMappingValue == null) {
@@ -235,8 +222,7 @@
testsToRunLabels,
targets.hasError(),
expandedTargets.hasError(),
- workspaceName.getName(),
- notSymlinkedInExecrootDirectories);
+ workspaceName.getName());
env.getListener()
.post(
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternPhaseValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternPhaseValue.java
index a1fa7dc..0de10a2 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternPhaseValue.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternPhaseValue.java
@@ -17,7 +17,6 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedSet;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
@@ -49,21 +48,18 @@
private final boolean hasError;
private final boolean hasPostExpansionError;
private final String workspaceName;
- private final ImmutableSortedSet<String> notSymlinkedInExecrootDirectories;
TargetPatternPhaseValue(
ImmutableSet<Label> targetLabels,
ImmutableSet<Label> testsToRunLabels,
boolean hasError,
boolean hasPostExpansionError,
- String workspaceName,
- ImmutableSortedSet<String> notSymlinkedInExecrootDirectories) {
+ String workspaceName) {
this.targetLabels = targetLabels;
this.testsToRunLabels = testsToRunLabels;
this.hasError = hasError;
this.hasPostExpansionError = hasPostExpansionError;
this.workspaceName = workspaceName;
- this.notSymlinkedInExecrootDirectories = notSymlinkedInExecrootDirectories;
}
/** Expensive. Results in a Skyframe evaluation. */
@@ -118,10 +114,6 @@
return workspaceName;
}
- public ImmutableSortedSet<String> getNotSymlinkedInExecrootDirectories() {
- return notSymlinkedInExecrootDirectories;
- }
-
@Override
public boolean equals(Object obj) {
if (this == obj) {
@@ -134,8 +126,6 @@
return Objects.equals(this.targetLabels, that.targetLabels)
&& Objects.equals(this.testsToRunLabels, that.testsToRunLabels)
&& Objects.equals(this.workspaceName, that.workspaceName)
- && Objects.equals(
- this.notSymlinkedInExecrootDirectories, that.notSymlinkedInExecrootDirectories)
&& this.hasError == that.hasError
&& this.hasPostExpansionError == that.hasPostExpansionError;
}
@@ -147,8 +137,7 @@
this.testsToRunLabels,
this.workspaceName,
this.hasError,
- this.hasPostExpansionError,
- this.notSymlinkedInExecrootDirectories);
+ this.hasPostExpansionError);
}
/** Create a target pattern phase value key. */
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java
index c8e8333..3ea17ab 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java
@@ -23,7 +23,6 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Sets;
import com.google.devtools.build.lib.actions.FileValue;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
@@ -252,8 +251,7 @@
workspaceFile,
/* idx = */ 0, // first fragment
/* hasNext = */ false,
- ImmutableMap.of(),
- ImmutableSortedSet.of());
+ ImmutableMap.of());
}
// Get the state at the end of the previous chunk.
@@ -357,8 +355,7 @@
workspaceFile,
key.getIndex(),
key.getIndex() < chunks.size() - 1,
- ImmutableMap.copyOf(parser.getManagedDirectories()),
- parser.getDoNotSymlinkInExecrootPaths());
+ ImmutableMap.copyOf(parser.getManagedDirectories()));
}
private static StarlarkFile parseWorkspaceFile(
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/AutoRegistry.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/AutoRegistry.java
index 6d80512..ce5638b 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/AutoRegistry.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/AutoRegistry.java
@@ -47,7 +47,6 @@
ImmutableList.of(
"com.google.devtools.build.lib.google",
"com.google.devtools.build.lib.vfs",
- "com.google.devtools.build.lib.bazel.rules.ninja",
"com.google.devtools.build.lib.actions.ArtifactFactory",
"com.google.devtools.build.lib.packages.PackageFactory$BuiltInRuleFunction",
"com.google.devtools.build.skyframe.SkyFunctionEnvironment");
diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/WorkspaceGlobalsApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/WorkspaceGlobalsApi.java
index 4d9d9ab..deae557 100644
--- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/WorkspaceGlobalsApi.java
+++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/WorkspaceGlobalsApi.java
@@ -16,7 +16,6 @@
package com.google.devtools.build.lib.starlarkbuildapi;
import com.google.devtools.build.docgen.annot.DocumentMethods;
-import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
import net.starlark.java.annot.Param;
import net.starlark.java.annot.ParamType;
import net.starlark.java.annot.StarlarkMethod;
@@ -83,34 +82,6 @@
throws EvalException, InterruptedException;
@StarlarkMethod(
- name = "toplevel_output_directories",
- doc =
- "Exclude directories under workspace from symlinking into execroot.\n"
- + "<p>Normally, source directories are symlinked to the execroot, so that the"
- + " actions can access the input (source) files.<p/><p>In the case of Ninja"
- + " execution (enabled with --experimental_ninja_actions flag), it is typical that"
- + " the directory with build-related files contains source files for the build, and"
- + " Ninja prescribes creation of the outputs in that same directory.</p><p>Since"
- + " commands in the Ninja file use relative paths to address source files and"
- + " directories, we must still allow the execution in the same-named directory under"
- + " the execroot. But we must avoid populating the underlying source directory with"
- + " output files.</p><p>This method can be used to specify that Ninja build"
- + " configuration directories should not be symlinked to the execroot. It is not"
- + " expected that there could be other use cases for using this method.</p>",
- parameters = {
- @Param(
- name = "paths",
- allowedTypes = {@ParamType(type = Sequence.class, generic1 = String.class)},
- doc = "",
- named = true,
- positional = false)
- },
- useStarlarkThread = true,
- enableOnlyWithFlag = BuildLanguageOptions.EXPERIMENTAL_NINJA_ACTIONS)
- void dontSymlinkDirectoriesInExecroot(Sequence<?> paths, StarlarkThread thread)
- throws EvalException, InterruptedException;
-
- @StarlarkMethod(
name = "register_execution_platforms",
doc =
"Register an already-defined platform so that Bazel can use it as an "
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestUtil.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestUtil.java
index 0e1e0a7..3cceb03 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestUtil.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestUtil.java
@@ -58,7 +58,6 @@
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionName;
import java.io.IOException;
@@ -156,11 +155,6 @@
}
@Override
- public Artifact getSourceArtifactForNinjaBuild(PathFragment execPath, Root root) {
- return original.getSourceArtifactForNinjaBuild(execPath, root);
- }
-
- @Override
public Artifact getFilesetArtifact(PathFragment rootRelativePath, ArtifactRoot root) {
return original.getFilesetArtifact(rootRelativePath, root);
}
@@ -381,11 +375,6 @@
}
@Override
- public Artifact getSourceArtifactForNinjaBuild(PathFragment execPath, Root root) {
- return null;
- }
-
- @Override
public ExtendedEventHandler getEventHandler() {
return null;
}
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
index cc2cdf7..86020b0 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
@@ -2217,11 +2217,6 @@
}
@Override
- public Artifact getSourceArtifactForNinjaBuild(PathFragment execPath, Root root) {
- throw new UnsupportedOperationException();
- }
-
- @Override
public ExtendedEventHandler getEventHandler() {
return reporter;
}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/rules/BUILD
index 62f7a3e..e6e2942 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/BUILD
@@ -12,7 +12,6 @@
"//src/test/java/com/google/devtools/build/lib/bazel/rules/android:srcs",
"//src/test/java/com/google/devtools/build/lib/bazel/rules/genrule:srcs",
"//src/test/java/com/google/devtools/build/lib/bazel/rules/java:srcs",
- "//src/test/java/com/google/devtools/build/lib/bazel/rules/ninja:srcs",
"//src/test/java/com/google/devtools/build/lib/bazel/rules/python:srcs",
"//src/test/java/com/google/devtools/build/lib/bazel/rules/sh:srcs",
],
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/BUILD
deleted file mode 100644
index 321250c..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/BUILD
+++ /dev/null
@@ -1,50 +0,0 @@
-load("@rules_java//java:defs.bzl", "java_library", "java_test")
-
-package(
- default_testonly = 1,
- default_visibility = ["//src:__subpackages__"],
-)
-
-filegroup(
- name = "srcs",
- testonly = 0,
- srcs = glob(["*"]),
- visibility = ["//src:__subpackages__"],
-)
-
-java_library(
- name = "NinjaTests_lib",
- srcs = glob(["*.java"]),
- deps = [
- "//src/main/java/com/google/devtools/build/lib/actions",
- "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
- "//src/main/java/com/google/devtools/build/lib/analysis:actions/symlink_action",
- "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
- "//src/main/java/com/google/devtools/build/lib/analysis:configured_target",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/actions",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/file",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/lexer",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/parser:parser_impl",
- "//src/main/java/com/google/devtools/build/lib/bazel/rules/ninja/pipeline:pipeline_impl",
- "//src/main/java/com/google/devtools/build/lib/concurrent",
- "//src/main/java/com/google/devtools/build/lib/util",
- "//src/main/java/com/google/devtools/build/lib/vfs",
- "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
- "//src/test/java/com/google/devtools/build/lib/actions/util",
- "//src/test/java/com/google/devtools/build/lib/analysis/util",
- "//src/test/java/com/google/devtools/build/lib/testutil",
- "//third_party:guava",
- "//third_party:jsr305",
- "//third_party:junit4",
- "//third_party:truth",
- ],
-)
-
-java_test(
- name = "NinjaTests",
- test_class = "com.google.devtools.build.lib.AllTests",
- runtime_deps = [
- ":NinjaTests_lib",
- "//src/test/java/com/google/devtools/build/lib:test_runner",
- ],
-)
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/DeclarationAssemblerTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/DeclarationAssemblerTest.java
deleted file mode 100644
index 855ea97..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/DeclarationAssemblerTest.java
+++ /dev/null
@@ -1,144 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja;
-
-import static com.google.common.truth.Truth.assertThat;
-import static java.nio.charset.StandardCharsets.ISO_8859_1;
-
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.DeclarationAssembler;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.util.Pair;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link DeclarationAssembler}. */
-@RunWith(JUnit4.class)
-public class DeclarationAssemblerTest {
- @Test
- public void testOneCharacterFragment() throws Exception {
- List<ByteBuffer> buffers = new ArrayList<>();
- buffers.add(ByteBuffer.wrap("abcdefghij\na".getBytes(ISO_8859_1)));
- buffers.add(ByteBuffer.wrap("bcdefg\n".getBytes(ISO_8859_1)));
-
- List<FileFragment> fragments = new ArrayList<>();
- fragments.add(new FileFragment(buffers.get(0), 0, 0, 11)); // abcdefghij\n
- fragments.add(new FileFragment(buffers.get(0), 0, 11, 12)); // a
- fragments.add(new FileFragment(buffers.get(1), 12, 0, 7)); // bcdefg\n
-
- List<String> result = new ArrayList<>();
- DeclarationAssembler assembler = new DeclarationAssembler(d -> result.add(d.toString()));
- assembler.wrapUp(fragments);
- assertThat(result).containsExactly("abcdefghij\n", "abcdefg\n").inOrder();
- }
-
- @Test
- public void testAssembleLines() throws Exception {
- // Glue two parts of the same token together
- doSameBufferTest("0123456789", 1, 3, 3, 5, "1234");
- // The '\n' symbol happened to be the last in the buffer, we should correctly
- // not merge two parts, but create two separate tokens
- doSameBufferTest("01\n3456789", 1, 3, 3, 5, "1\n", "34");
- // The "\n " sequence does not separate a new token, because of the starting space
- doSameBufferTest("01\n 3456789", 1, 4, 4, 6, "1\n 34");
-
- doTwoBuffersTest("abc", "def", "abcdef");
- doTwoBuffersTest("abc\n", "def", "abc\n", "def");
- doTwoBuffersTest("abc\n", " def", "abc\n def");
- doTwoBuffersTest("abc", "\ndef", "abc\n", "def");
-
- doTwoBuffersTest("abc$\n", "def", "abc$\ndef");
- doTwoBuffersTest("abc$", "\ndef", "abc$\ndef");
- }
-
- @Test
- public void testMergeTwoDifferentBuffers() throws Exception {
- List<Pair<Long, String>> offsetStringPairList = Lists.newArrayList();
- String unrelatedFirstBuffer = " ".repeat(100);
- String s1 = "hello";
- String s2 = "goodbye";
- byte[] chars1 = (unrelatedFirstBuffer + s1).getBytes(ISO_8859_1);
- ByteBuffer bytes1 = ByteBuffer.wrap(chars1);
- byte[] chars2 = s2.getBytes(ISO_8859_1);
- ByteBuffer bytes2 = ByteBuffer.wrap(chars2);
-
- DeclarationAssembler assembler =
- new DeclarationAssembler(
- fragment -> {
- offsetStringPairList.add(
- new Pair<>(fragment.getFragmentOffset(), fragment.toString()));
- });
-
- assembler.wrapUp(
- Lists.newArrayList(
- new FileFragment(bytes1, 0, unrelatedFirstBuffer.length(), chars1.length),
- new FileFragment(bytes2, chars1.length, 0, s2.length())));
-
- assertThat(Iterables.getOnlyElement(offsetStringPairList))
- .isEqualTo(new Pair<>((long) unrelatedFirstBuffer.length(), "hellogoodbye"));
- }
-
- private static void doTwoBuffersTest(String s1, String s2, String... expected)
- throws GenericParsingException, IOException {
- List<String> list = Lists.newArrayList();
- final byte[] chars1 = s1.getBytes(ISO_8859_1);
- ByteBuffer bytes1 = ByteBuffer.wrap(chars1);
- final byte[] chars2 = s2.getBytes(ISO_8859_1);
- ByteBuffer bytes2 = ByteBuffer.wrap(chars2);
-
- DeclarationAssembler assembler =
- new DeclarationAssembler(
- fragment -> {
- list.add(fragment.toString());
- assertThat(fragment.getFileOffset()).isAnyOf(0L, (long) chars1.length);
- });
-
- assembler.wrapUp(
- Lists.newArrayList(
- new FileFragment(bytes1, 0, 0, s1.length()),
- new FileFragment(bytes2, chars1.length, 0, s2.length())));
-
- assertThat(list).isEqualTo(Arrays.asList(expected));
- }
-
- private static void doSameBufferTest(
- String s, int start1, int end1, int start2, int end2, String... expected)
- throws GenericParsingException, IOException {
- List<String> list = Lists.newArrayList();
- DeclarationAssembler assembler =
- new DeclarationAssembler(
- fragment -> {
- list.add(fragment.toString());
- assertThat(fragment.getFileOffset()).isEqualTo(0);
- });
-
- byte[] chars = s.getBytes(ISO_8859_1);
- ByteBuffer bytes = ByteBuffer.wrap(chars);
- assembler.wrapUp(
- Lists.newArrayList(
- new FileFragment(bytes, 0, start1, end1), new FileFragment(bytes, 0, start2, end2)));
-
- assertThat(list).isEqualTo(Arrays.asList(expected));
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/FileFragmentSplitterTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/FileFragmentSplitterTest.java
deleted file mode 100644
index 41d08c3..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/FileFragmentSplitterTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja;
-
-import static com.google.common.truth.Truth.assertThat;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.DeclarationConsumer;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragmentSplitter;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.stream.Collectors;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link FileFragmentSplitter} */
-@RunWith(JUnit4.class)
-public class FileFragmentSplitterTest {
- @Test
- public void testTokenizeSimple() throws Exception {
- List<String> list = ImmutableList.of("one", "two", "three");
-
- List<String> result = Lists.newArrayList();
- int offsetValue = 123;
- DeclarationConsumer consumer =
- fragment -> {
- result.add(fragment.toString());
- assertThat(fragment.getFileOffset()).isEqualTo(offsetValue);
- };
-
- byte[] chars = String.join("\n", list).getBytes(StandardCharsets.ISO_8859_1);
- ByteBuffer buffer = ByteBuffer.wrap(chars);
- FileFragment fragment = new FileFragment(buffer, offsetValue, 0, chars.length);
- FileFragmentSplitter tokenizer = new FileFragmentSplitter(fragment, consumer);
- List<FileFragment> edges = tokenizer.call();
- assertThat(result).containsExactly("two\n");
- assertThat(
- edges.stream()
- .map(pair -> Preconditions.checkNotNull(pair).toString())
- .collect(Collectors.toList()))
- .containsExactly("one\n", "three")
- .inOrder();
- }
-
- @Test
- public void testTokenizeWithDetails() throws Exception {
- List<String> list =
- ImmutableList.of("one", " one-detail", "two", "\ttwo-detail", "three", " three-detail");
- byte[] chars = String.join("\n", list).getBytes(StandardCharsets.ISO_8859_1);
- ByteBuffer bytes = ByteBuffer.wrap(chars);
-
- List<String> result = Lists.newArrayList();
- DeclarationConsumer consumer = fragment -> result.add(fragment.toString());
-
- FileFragment fragment = new FileFragment(bytes, 0, 0, chars.length);
- FileFragmentSplitter tokenizer = new FileFragmentSplitter(fragment, consumer);
- List<FileFragment> edges = tokenizer.call();
- assertThat(result).containsExactly("two\n\ttwo-detail\n");
- assertThat(
- edges.stream()
- .map(pair -> Preconditions.checkNotNull(pair).toString())
- .collect(Collectors.toList()))
- .containsExactly("one\n one-detail\n", "three\n three-detail")
- .inOrder();
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/FileFragmentTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/FileFragmentTest.java
deleted file mode 100644
index ad7374e..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/FileFragmentTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja;
-
-import static com.google.common.truth.Truth.assertThat;
-
-import com.google.common.collect.ImmutableList;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link FileFragment} */
-@RunWith(JUnit4.class)
-public class FileFragmentTest {
- @Test
- public void testMethods() {
- final byte[] bytes = "0123456789".getBytes(StandardCharsets.ISO_8859_1);
- FileFragment fragment = new FileFragment(ByteBuffer.wrap(bytes), 0, 1, 9);
- assertThat(fragment.length()).isEqualTo(8);
- assertThat(fragment.toString()).isEqualTo("12345678");
- assertThat(fragment.subFragment(2, 4).toString()).isEqualTo("34");
- assertThat(fragment.subFragment(0, 8).toString()).isEqualTo("12345678");
- }
-
- @Test
- public void testMerge() {
- final byte[] bytes = "0123456789".getBytes(StandardCharsets.ISO_8859_1);
- FileFragment first = new FileFragment(ByteBuffer.wrap(bytes), 0, 1, 9);
- final byte[] abcBytes = "abcdefg".getBytes(StandardCharsets.ISO_8859_1);
- FileFragment second = new FileFragment(ByteBuffer.wrap(abcBytes), bytes.length, 1, 4);
-
- assertThat(FileFragment.merge(ImmutableList.of(first))).isSameInstanceAs(first);
- FileFragment merged = FileFragment.merge(ImmutableList.of(first, second));
- assertThat(merged.length()).isEqualTo(11);
- assertThat(merged.toString()).isEqualTo("12345678bcd");
- }
-
- @Test
- public void testEscapeCharacters() {
- final byte[] bytes = "\0\n\t\r".getBytes(StandardCharsets.ISO_8859_1);
- FileFragment fragment = new FileFragment(ByteBuffer.wrap(bytes), 0, 0, 3);
- assertThat(fragment.length()).isEqualTo(3);
- assertThat(fragment.toString()).isEqualTo("\0\n\t");
- assertThat(fragment.subFragment(1, 3).toString()).isEqualTo("\n\t");
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaBuildTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaBuildTest.java
deleted file mode 100644
index 57e0138..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaBuildTest.java
+++ /dev/null
@@ -1,860 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja;
-
-import static com.google.common.truth.Truth.assertThat;
-import static org.junit.Assert.assertThrows;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
-import com.google.devtools.build.lib.actions.Artifact;
-import com.google.devtools.build.lib.actions.CommandLines.CommandLineAndParamFileInfo;
-import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
-import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
-import com.google.devtools.build.lib.analysis.ConfiguredTarget;
-import com.google.devtools.build.lib.analysis.OutputGroupInfo;
-import com.google.devtools.build.lib.analysis.actions.SymlinkAction;
-import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget;
-import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
-import com.google.devtools.build.lib.bazel.rules.ninja.actions.NinjaAction;
-import com.google.devtools.build.lib.bazel.rules.ninja.actions.NinjaBuildRule;
-import com.google.devtools.build.lib.bazel.rules.ninja.actions.NinjaGraphRule;
-import com.google.devtools.build.lib.testutil.TestRuleClassProvider;
-import java.util.List;
-import java.util.stream.Collectors;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Test for the {@code NinjaBuild} configured target factory. */
-@RunWith(JUnit4.class)
-public class NinjaBuildTest extends BuildViewTestCase {
-
- @Override
- protected ConfiguredRuleClassProvider createRuleClassProvider() {
- ConfiguredRuleClassProvider.Builder builder = new ConfiguredRuleClassProvider.Builder();
- TestRuleClassProvider.addStandardRules(builder);
- builder.addRuleDefinition(new NinjaGraphRule());
- builder.addRuleDefinition(new NinjaBuildRule());
- return builder.build();
- }
-
- @Before
- public void setUp() throws Exception {
- setBuildLanguageOptions("--experimental_ninja_actions");
- }
-
- @Test
- public void testSourceFileNotInSubtree() throws Exception {
- rewriteWorkspace("toplevel_output_directories(paths=['out'])");
-
- scratch.file("a/n.ninja", "rule cp", " command = cp $in $out", "build out/o: cp subdir/i");
-
- scratch.file(
- "a/BUILD",
- "ninja_graph(name='graph', output_root='out', main='n.ninja')",
- "ninja_build(name='build', ninja_graph=':graph', output_groups={'o': ['out/o']})");
-
- reporter.removeHandler(failFastHandler);
- getConfiguredTarget("//a:build");
- assertContainsEvent(
- "Source artifact 'subdir/i' is not under the package directory 'a' of ninja_build rule");
- }
-
- @Test
- public void testNinjaBuildRule() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("build_config/input.txt", "World");
- scratch.file(
- "build_config/build.ninja",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- " description = Creating ${out}",
- "build build_config/hello.txt: echo build_config/input.txt");
-
- // Working directory is workspace root.
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " main = 'build_config/build.ninja',",
- " output_root_inputs = ['input.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['build_config/hello.txt']})");
- assertThat(configuredTarget).isInstanceOf(RuleConfiguredTarget.class);
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- assertThat(actions).hasSize(1);
- ActionAnalysisMetadata action = Iterables.getOnlyElement(actions);
- assertThat(action).isInstanceOf(NinjaAction.class);
- NinjaAction ninjaAction = (NinjaAction) action;
- List<CommandLineAndParamFileInfo> commandLines =
- ninjaAction.getCommandLines().getCommandLines();
- assertThat(commandLines).hasSize(1);
- assertThat(commandLines.get(0).commandLine.toString())
- .endsWith("echo \"Hello $(cat build_config/input.txt)!\" > build_config/hello.txt");
- assertThat(ninjaAction.getPrimaryInput().getExecPathString())
- .isEqualTo("build_config/input.txt");
- assertThat(ninjaAction.getPrimaryOutput().getExecPathString())
- .isEqualTo("build_config/hello.txt");
- assertThat(ninjaAction.getProgressMessage()).isEqualTo("Creating build_config/hello.txt");
- }
-
- @Test
- public void testNinjaBuildRule_progressMessageFromRule() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("build_config/input.txt", "World");
- scratch.file(
- "build_config/build.ninja",
- "file_variable = with greetings",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- " description = ${action} ${out} ${file_variable}",
- "build build_config/hello.txt: echo build_config/input.txt",
- " action = Creating");
-
- // Working directory is workspace root.
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " main = 'build_config/build.ninja',",
- " output_root_inputs = ['input.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['build_config/hello.txt']})");
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- NinjaAction ninjaAction = (NinjaAction) Iterables.getOnlyElement(actions);
-
- // The rule description is expanded with rule, build and file level variables.
- assertThat(ninjaAction.getProgressMessage())
- .isEqualTo("Creating build_config/hello.txt with greetings");
- }
-
- @Test
- public void testNinjaBuildRule_progressMessageFromBuildStatement() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("build_config/input.txt", "World");
- scratch.file(
- "build_config/build.ninja",
- "file_variable = foo bar baz",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- " description = Creating ${out}",
- "build build_config/hello.txt: echo build_config/input.txt",
- " description = ${file_variable} qux ");
-
- // Working directory is workspace root.
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " main = 'build_config/build.ninja',",
- " output_root_inputs = ['input.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['build_config/hello.txt']})");
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- NinjaAction ninjaAction = (NinjaAction) Iterables.getOnlyElement(actions);
-
- // The build description is expanded with the file level variable.
- assertThat(ninjaAction.getProgressMessage()).isEqualTo("foo bar baz qux ");
- }
-
- @Test
- public void testNinjaBuildRule_progressMessageGeneratedAtRuntime() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("build_config/input.txt", "World");
- scratch.file(
- "build_config/build.ninja",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- "build build_config/hello.txt: echo build_config/input.txt");
-
- // Working directory is workspace root.
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " main = 'build_config/build.ninja',",
- " output_root_inputs = ['input.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['build_config/hello.txt']})");
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- NinjaAction ninjaAction = (NinjaAction) Iterables.getOnlyElement(actions);
-
- // No description in either rule or build statements, so pretty print with rule name and output
- // basenames.
- assertThat(ninjaAction.getProgressMessage()).isEqualTo("[rule echo] Outputs: hello.txt");
- }
-
- @Test
- public void testNinjaGraphRuleWithPhonyTarget() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("build_config/input.txt", "World");
- scratch.file(
- "build_config/build.ninja",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- "build hello.txt: echo input.txt",
- "build alias: phony hello.txt");
-
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " working_directory = 'build_config',",
- " main = 'build_config/build.ninja',",
- " output_root_inputs = ['input.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['alias']})");
- assertThat(configuredTarget).isInstanceOf(RuleConfiguredTarget.class);
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- ActionAnalysisMetadata action = Iterables.getOnlyElement(actions);
-
- assertThat(actions).hasSize(1);
- assertThat(action).isInstanceOf(NinjaAction.class);
- NinjaAction ninjaAction = (NinjaAction) action;
- List<CommandLineAndParamFileInfo> commandLines =
- ninjaAction.getCommandLines().getCommandLines();
- assertThat(commandLines).hasSize(1);
- assertThat(commandLines.get(0).commandLine.toString())
- .endsWith("cd build_config && echo \"Hello $(cat input.txt)!\" > hello.txt");
- assertThat(ninjaAction.getPrimaryInput().getExecPathString())
- .isEqualTo("build_config/input.txt");
- assertThat(ninjaAction.getPrimaryOutput().getExecPathString())
- .isEqualTo("build_config/hello.txt");
- assertThat(ninjaAction.getProgressMessage()).isEqualTo("[rule echo] Outputs: hello.txt");
- }
-
- @Test
- public void testNinjaGraphRuleWithPhonyTree() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("build_config/a.txt", "A");
- scratch.file("build_config/b.txt", "B");
- scratch.file("build_config/c.txt", "C");
- scratch.file("build_config/d.txt", "D");
- scratch.file("build_config/e.txt", "E");
-
- scratch.file(
- "build_config/build.ninja",
- "rule cat",
- " command = cat ${in} > ${out}",
- "rule echo",
- " command = echo \"Hello $$(cat ${in} | tr '\\r\\n' ' ')!\" > ${out}",
- "build a: cat a.txt",
- "build b: cat b.txt",
- "build c: cat c.txt",
- "build d: cat d.txt",
- // e should be executed unconditionally as it depends on always-dirty phony action
- "build e: cat e.txt always_dirty",
- "build always_dirty: phony",
- "build group1: phony a b c",
- "build group2: phony d e",
- "build inputs_alias: phony group1 group2",
- "build hello.txt: echo inputs_alias",
- "build alias: phony hello.txt");
-
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " working_directory = 'build_config',",
- " main = 'build_config/build.ninja',",
- " output_root_inputs = ['a.txt', 'b.txt', 'c.txt', 'd.txt', 'e.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['alias']})");
- assertThat(configuredTarget).isInstanceOf(RuleConfiguredTarget.class);
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- assertThat(actions).hasSize(6);
- List<String> outputs = Lists.newArrayList();
- actions.forEach(a -> outputs.add(Iterables.getOnlyElement(a.getOutputs()).getExecPathString()));
- assertThat(outputs)
- .containsExactlyElementsIn(
- new String[] {
- "build_config/hello.txt",
- "build_config/a",
- "build_config/b",
- "build_config/c",
- "build_config/d",
- "build_config/e"
- });
-
- for (ActionAnalysisMetadata action : actions) {
- Artifact artifact = action.getPrimaryOutput();
- if ("hello.txt".equals(artifact.getFilename())) {
- assertThat(action).isInstanceOf(NinjaAction.class);
- NinjaAction ninjaAction = (NinjaAction) action;
- List<CommandLineAndParamFileInfo> commandLines =
- ninjaAction.getCommandLines().getCommandLines();
- assertThat(commandLines).hasSize(1);
- assertThat(commandLines.get(0).commandLine.toString())
- .contains(
- "cd build_config && echo \"Hello $(cat inputs_alias | tr '\\r\\n' ' ')!\""
- + " > hello.txt");
- List<String> inputPaths =
- ninjaAction.getInputs().toList().stream()
- .map(Artifact::getExecPathString)
- .collect(Collectors.toList());
- assertThat(inputPaths)
- .containsExactly(
- "build_config/a",
- "build_config/b",
- "build_config/c",
- "build_config/d",
- "build_config/e");
- assertThat(ninjaAction.getPrimaryOutput().getExecPathString())
- .isEqualTo("build_config/hello.txt");
- } else if ("e".equals(artifact.getFilename())) {
- assertThat(action).isInstanceOf(NinjaAction.class);
- NinjaAction ninjaAction = (NinjaAction) action;
- List<CommandLineAndParamFileInfo> commandLines =
- ninjaAction.getCommandLines().getCommandLines();
- assertThat(commandLines).hasSize(1);
- assertThat(commandLines.get(0).commandLine.toString())
- .endsWith("cd build_config && cat e.txt always_dirty > e");
- assertThat(ninjaAction.executeUnconditionally()).isTrue();
- }
- }
- }
-
- @Test
- public void testDepsMapping() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("input.txt", "World");
- scratch.file(
- "build_config/build.ninja",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- "build hello.txt: echo placeholder");
-
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " working_directory = 'build_config',",
- " main = 'build_config/build.ninja')",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['hello.txt']},",
- " deps_mapping = {'placeholder': ':input.txt'})");
- assertThat(configuredTarget).isInstanceOf(RuleConfiguredTarget.class);
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- assertThat(actions).hasSize(2);
-
- ActionAnalysisMetadata symlinkAction = actions.get(0);
- assertThat(symlinkAction).isInstanceOf(SymlinkAction.class);
- assertThat(symlinkAction.getPrimaryInput().getExecPathString()).isEqualTo("input.txt");
- assertThat(symlinkAction.getPrimaryOutput().getExecPathString())
- .isEqualTo("build_config/placeholder");
-
- ActionAnalysisMetadata action = actions.get(1);
- assertThat(action).isInstanceOf(NinjaAction.class);
- NinjaAction ninjaAction = (NinjaAction) action;
- List<CommandLineAndParamFileInfo> commandLines =
- ninjaAction.getCommandLines().getCommandLines();
- assertThat(commandLines).hasSize(1);
- assertThat(commandLines.get(0).commandLine.toString())
- .endsWith("cd build_config && echo \"Hello $(cat placeholder)!\" > hello.txt");
- assertThat(ninjaAction.getPrimaryInput().getExecPathString())
- .isEqualTo("build_config/placeholder");
- assertThat(ninjaAction.getPrimaryOutput().getExecPathString())
- .isEqualTo("build_config/hello.txt");
- }
-
- @Test
- public void testOnlySubGraphIsCreated() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("build_config/a.txt", "A");
- scratch.file("build_config/b.txt", "B");
- scratch.file("build_config/c.txt", "C");
- scratch.file("build_config/d.txt", "D");
- scratch.file("build_config/e.txt", "E");
-
- scratch.file(
- "build_config/build.ninja",
- "rule cat",
- " command = cat ${in} > ${out}",
- "rule echo",
- " command = echo \"Hello $$(cat ${in} | tr '\\r\\n' ' ')!\" > ${out}",
- "build a: cat a.txt",
- "build b: cat b.txt",
- "build c: cat c.txt",
- "build d: cat d.txt",
- "build e: cat e.txt",
- "build group1: phony a b c",
- "build group2: phony d e",
- "build inputs_alias: phony group1 group2",
- "build hello.txt: echo inputs_alias",
- "build alias: phony hello.txt");
-
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " working_directory = 'build_config',",
- " main = 'build_config/build.ninja',",
- " output_root_inputs = ['a.txt', 'b.txt', 'c.txt', 'd.txt', 'e.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['group1']})");
- assertThat(configuredTarget).isInstanceOf(RuleConfiguredTarget.class);
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- assertThat(actions).hasSize(3);
- List<String> outputs = Lists.newArrayList();
- actions.forEach(a -> outputs.add(Iterables.getOnlyElement(a.getOutputs()).getExecPathString()));
- assertThat(outputs)
- .containsExactlyElementsIn(
- new String[] {
- "build_config/a", "build_config/b", "build_config/c",
- });
- }
-
- @Test
- public void testRuleWithDepfileVariable() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("input");
- scratch.file(
- "build_config/build.ninja",
- "rule rule123",
- " command = executable -d ${depfile} ${in} > ${out}",
- " depfile = ${out}.d",
- " deps = gcc",
- "build out_file: rule123 ../input");
-
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " working_directory = 'build_config',",
- " main = 'build_config/build.ninja')",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['out_file']})");
- assertThat(configuredTarget).isInstanceOf(RuleConfiguredTarget.class);
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- assertThat(actions).hasSize(1);
-
- ActionAnalysisMetadata action = Iterables.getOnlyElement(actions);
- assertThat(action).isInstanceOf(NinjaAction.class);
- List<CommandLineAndParamFileInfo> commandLines =
- ((NinjaAction) action).getCommandLines().getCommandLines();
- assertThat(commandLines).hasSize(1);
- assertThat(commandLines.get(0).commandLine.toString())
- .endsWith("cd build_config && executable -d out_file.d ../input > out_file");
-
- assertThat(ActionsTestUtil.baseArtifactNames(action.getOutputs()))
- .containsExactly("out_file", "out_file.d");
- }
-
- @Test
- public void testCreateOutputSymlinkArtifactsInTopLevelDirectory() throws Exception {
- // output root: <cwd>/out/
- // working directory: <cwd>
- rewriteWorkspace("workspace(name = 'test')", "toplevel_output_directories(paths = ['out'])");
-
- scratch.file(
- "out/build.ninja",
- "rule symlink_rule",
- " command = ln -s fictive-file ${out}",
- " symlink_outputs = $out",
- "build out/dangling_symlink: symlink_rule");
-
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'out',",
- " main = 'out/build.ninja')",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['out/dangling_symlink']})");
- assertThat(configuredTarget).isInstanceOf(RuleConfiguredTarget.class);
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- assertThat(actions).hasSize(1);
-
- ActionAnalysisMetadata action = Iterables.getOnlyElement(actions);
- Artifact primaryOutput = action.getPrimaryOutput();
- assertThat(primaryOutput.isSymlink()).isTrue();
- assertThat(action).isInstanceOf(NinjaAction.class);
-
- List<CommandLineAndParamFileInfo> commandLines =
- ((NinjaAction) action).getCommandLines().getCommandLines();
- assertThat(commandLines).hasSize(1);
- assertThat(commandLines.get(0).commandLine.toString())
- .endsWith("ln -s fictive-file out/dangling_symlink");
- }
-
- @Test
- public void testCreateOutputSymlinkArtifactsInWorkingDirectory() throws Exception {
- // output root: <cwd>/build_config
- // working directory: <cwd>/build_config
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file(
- "build_config/build.ninja",
- "rule symlink_rule",
- " command = ln -s fictive-file ${out}",
- " symlink_outputs = $out",
- "build dangling_symlink: symlink_rule");
-
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " working_directory = 'build_config',",
- " main = 'build_config/build.ninja')",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['dangling_symlink']})");
- assertThat(configuredTarget).isInstanceOf(RuleConfiguredTarget.class);
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- assertThat(actions).hasSize(1);
-
- ActionAnalysisMetadata action = Iterables.getOnlyElement(actions);
- Artifact primaryOutput = action.getPrimaryOutput();
- assertThat(primaryOutput.isSymlink()).isTrue();
- assertThat(action).isInstanceOf(NinjaAction.class);
-
- List<CommandLineAndParamFileInfo> commandLines =
- ((NinjaAction) action).getCommandLines().getCommandLines();
- assertThat(commandLines).hasSize(1);
- assertThat(commandLines.get(0).commandLine.toString())
- .endsWith("cd build_config && ln -s fictive-file dangling_symlink");
- }
-
- @Test
- public void testCreateIntermediateOutputSymlinkArtifacts() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file(
- "build_config/build.ninja",
- "rule symlink_rule",
- " command = ln -s fictive-file ${out}",
- "rule cat",
- " command = cat ${in} > ${out}",
- "build dangling_symlink: symlink_rule",
- " symlink_outputs = dangling_symlink",
- "build mybuild: cat dangling_symlink");
-
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " working_directory = 'build_config',",
- " main = 'build_config/build.ninja')",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['mybuild']})");
- assertThat(configuredTarget).isInstanceOf(RuleConfiguredTarget.class);
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- assertThat(actions).hasSize(2);
-
- ActionAnalysisMetadata symlinkAction = actions.get(1);
- Artifact primaryOutput = symlinkAction.getPrimaryOutput();
- assertThat(primaryOutput.isSymlink()).isTrue();
- assertThat(symlinkAction).isInstanceOf(NinjaAction.class);
-
- List<CommandLineAndParamFileInfo> commandLines =
- ((NinjaAction) symlinkAction).getCommandLines().getCommandLines();
- assertThat(commandLines).hasSize(1);
- assertThat(commandLines.get(0).commandLine.toString())
- .endsWith("cd build_config && ln -s fictive-file dangling_symlink");
- }
-
- @Test
- public void testOutputRootInputsWithConflictingNinjaActionOutput() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("build_config/hello.txt", "hello");
- scratch.file(
- "build_config/build.ninja",
- "rule hello_world",
- " command = echo \"Hello World!\" > ${out}",
- "build build_config/hello.txt: hello_world",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- "build build_config/out.txt: echo build_config/hello.txt");
-
- // Working directory is workspace root.
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " main = 'build_config/build.ninja',",
- // hello.txt will be symlinked using a symlink action, but will also be an output of
- // the NinjaAction created from the ninja rule above.
- " output_root_inputs = ['hello.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['build_config/out.txt']})");
-
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- // The build.ninja file has rules for two actions, but only one of them should have been
- // registered. Normally this would produce an ActionsConflictException, but we skip the action.
- assertThat(actions).hasSize(1);
- assertThat(actions.get(0).getOutputs().asList().get(0).getExecPathString())
- .isEqualTo("build_config/out.txt");
- }
-
- @Test
- public void testOutputRootInputsWithConflictingNinjaActionOutputThrowsErrorOnNonSymlinkOutput()
- throws Exception {
-
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("build_config/hello.txt", "hello");
- scratch.file(
- "build_config/build.ninja",
- "rule hello_world",
- " command = echo \"Hello World!\" > ${out}",
- "build build_config/hello.txt build_config/not_an_input_symlink.txt: hello_world",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- "build build_config/out.txt: echo build_config/hello.txt");
-
- String message =
- "in ninja_build rule //:ninja_target: Ninja target hello_world has outputs in "
- + "output_root_inputs and other outputs not in output_root_inputs:\n"
- + "Outputs in output_root_inputs:\n"
- + " build_config/hello.txt\n"
- + "Outputs not in output_root_inputs:\n"
- + " build_config/not_an_input_symlink.txt";
-
- // A GenericParsingException is what's actually created, however the rule code and the
- // testing framework only relays the exception's messasge wrapped in an AssertionError.
- Throwable throwable =
- assertThrows(
- AssertionError.class,
- () ->
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " main = 'build_config/build.ninja',",
- // hello.txt will be symlinked using a symlink action, but will also be an
- // output of
- // the NinjaAction created from the ninja rule above.
- " output_root_inputs = ['hello.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['build_config/out.txt']})"));
- assertThat(throwable).hasMessageThat().contains(message);
- }
-
- /**
- * Tests that outputs from validation actions are correctly read from Ninja files and added to the
- * validation output groups. Note that validation inputs syntax ("|@") is specific to AOSP's
- * implementation of Ninja.
- */
- @Test
- public void testNinjaValidationInputs() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("build_config/input.txt", "World");
- scratch.file("build_config/validation_input.txt", "6\n7\n8");
- scratch.file(
- "build_config/build.ninja",
- "rule validate",
- " command = grep 7 ${in} > ${out}",
- " description = Validating input",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- " description = Creating ${out}",
- "build build_config/validation_output.txt: validate build_config/validation_input.txt",
- "build build_config/hello.txt: echo build_config/input.txt "
- + "|@ build_config/validation_output.txt",
- "build build_config/hello2.txt: echo build_config/input.txt");
-
- // Working directory is workspace root.
- RuleConfiguredTarget configuredTarget =
- (RuleConfiguredTarget)
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " main = 'build_config/build.ninja',",
- " output_root_inputs = ['input.txt', 'validation_input.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups = {'main': ['build_config/hello.txt']})",
- "ninja_build(name = 'ninja_target2', ninja_graph = 'graph',",
- " output_groups = {'main': ['build_config/hello2.txt']})");
-
- OutputGroupInfo outputGroupInfo = OutputGroupInfo.get(configuredTarget);
- List<Artifact> validationArtifacts =
- outputGroupInfo.getOutputGroup(OutputGroupInfo.VALIDATION).toList();
- assertThat(validationArtifacts).hasSize(1);
- assertThat(validationArtifacts.get(0).getExecPathString())
- .isEqualTo("build_config/validation_output.txt");
-
- ActionAnalysisMetadata echoAction = getGeneratingAction(configuredTarget, "hello.txt");
-
- // Sanity check that we're specifying the paths correctly and double check that we have the
- // right action.
- assertThat(actionInputsToPaths(echoAction.getInputs())).contains("build_config/input.txt");
- // The validation output should not be in the direct inputs of the action which depends on it,
- // so that this action is not blocked on generating that validation input.
- assertThat(actionInputsToPaths(echoAction.getInputs()))
- .doesNotContain("build_config/validation_output.txt");
-
- // ninja_target2 does not have a ninja build rule with a validation input, so it should not
- // have any validation outputs in its validation output group.
- RuleConfiguredTarget targetNoValidations =
- (RuleConfiguredTarget) getConfiguredTarget("//:ninja_target2");
- assertThat(
- OutputGroupInfo.get(targetNoValidations)
- .getOutputGroup(OutputGroupInfo.VALIDATION)
- .toList())
- .isEmpty();
- }
-
- @Test
- public void testNinjaTransitiveValidationInputs() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("build_config/input.txt", "World");
- scratch.file("build_config/validation_input.txt", "6\n7\n8");
- scratch.file(
- "build_config/build.ninja",
- "rule validate",
- " command = grep 7 ${in} > ${out}",
- " description = Validating input",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- " description = Creating ${out}",
- "build build_config/validation_output.txt: validate build_config/validation_input.txt",
- "build build_config/hello.txt: echo build_config/input.txt "
- + "|@ build_config/validation_output.txt",
- "build build_config/hello2.txt: echo build_config/hello.txt");
-
- // Working directory is workspace root.
- RuleConfiguredTarget configuredTarget =
- (RuleConfiguredTarget)
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " main = 'build_config/build.ninja',",
- " output_root_inputs = ['input.txt', 'validation_input.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- // requesting hello2, whose generating action itself does not have a validation
- // action
- " output_groups = {'main': ['build_config/hello2.txt']})");
-
- OutputGroupInfo outputGroupInfo = OutputGroupInfo.get(configuredTarget);
- List<Artifact> validationArtifacts =
- outputGroupInfo.getOutputGroup(OutputGroupInfo.VALIDATION).toList();
- assertThat(validationArtifacts).hasSize(1);
- // The validation output group should still have the output of the validation action even if
- // that action is only in the transitive deps of the requested top-level artifact.
- assertThat(validationArtifacts.get(0).getExecPathString())
- .isEqualTo("build_config/validation_output.txt");
- }
-
- /**
- * Tests that the validation outputs from two independent build graphs in a ninja file are all
- * registered in the validation output group when those graphs are referenced from a nina_build
- * target.
- */
- @Test
- public void testNinjaValidationInputsIndependentValidationsAdded() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("build_config/input.txt", "World");
- scratch.file("build_config/validation_input.txt", "6\n7\n8");
- scratch.file("build_config/validation_input2.txt", "9\n0\n1");
- scratch.file(
- "build_config/build.ninja",
- "rule validate",
- " command = grep 7 ${in} > ${out}",
- " description = Validating input",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- " description = Creating ${out}",
- // two validation outputs
- "build build_config/validation_output.txt: validate build_config/validation_input.txt",
- "build build_config/validation_output2.txt: validate build_config/validation_input2.txt",
- // two build rules that consume validation inputs
- "build build_config/hello.txt: echo build_config/input.txt "
- + "|@ build_config/validation_output.txt",
- "build build_config/hello2.txt: echo build_config/input.txt "
- + "|@ build_config/validation_output2.txt");
-
- // Working directory is workspace root.
- RuleConfiguredTarget configuredTarget =
- (RuleConfiguredTarget)
- scratchConfiguredTarget(
- "",
- "ninja_target",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " main = 'build_config/build.ninja',",
- " output_root_inputs = ['input.txt', 'validation_input.txt',"
- + " 'validation_input2.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups = {",
- " 'out1': ['build_config/hello.txt'],",
- " 'out2': ['build_config/hello2.txt'],",
- " })");
-
- OutputGroupInfo outputGroupInfo = OutputGroupInfo.get(configuredTarget);
- List<Artifact> validationArtifacts =
- outputGroupInfo.getOutputGroup(OutputGroupInfo.VALIDATION).toList();
- assertThat(ActionsTestUtil.execPaths(validationArtifacts))
- .containsExactly(
- "build_config/validation_output.txt", "build_config/validation_output2.txt");
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaGraphTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaGraphTest.java
deleted file mode 100644
index 943523a..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaGraphTest.java
+++ /dev/null
@@ -1,284 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja;
-
-import static com.google.common.collect.ImmutableList.toImmutableList;
-import static com.google.common.truth.Truth.assertThat;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
-import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
-import com.google.devtools.build.lib.analysis.ConfiguredTarget;
-import com.google.devtools.build.lib.analysis.actions.SymlinkAction;
-import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget;
-import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
-import com.google.devtools.build.lib.bazel.rules.ninja.actions.NinjaGraphProvider;
-import com.google.devtools.build.lib.bazel.rules.ninja.actions.NinjaGraphRule;
-import com.google.devtools.build.lib.bazel.rules.ninja.actions.PhonyTarget;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget;
-import com.google.devtools.build.lib.testutil.TestRuleClassProvider;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.util.List;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link com.google.devtools.build.lib.bazel.rules.ninja.actions.NinjaGraphRule} */
-@RunWith(JUnit4.class)
-public class NinjaGraphTest extends BuildViewTestCase {
-
- @Override
- protected ConfiguredRuleClassProvider createRuleClassProvider() {
- ConfiguredRuleClassProvider.Builder builder = new ConfiguredRuleClassProvider.Builder();
- TestRuleClassProvider.addStandardRules(builder);
- builder.addRuleDefinition(new NinjaGraphRule());
- return builder.build();
- }
-
- @Before
- public void setUp() throws Exception {
- setBuildLanguageOptions("--experimental_ninja_actions");
- }
-
- @Test
- public void testNinjaGraphRule() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("build_config/input.txt", "World");
- scratch.file(
- "build_config/build.ninja",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- "build build_config/hello.txt: echo build_config/input.txt");
-
- // Working directory is workspace root.
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "graph",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " main = 'build_config/build.ninja',",
- " output_root_inputs = ['input.txt'])");
- assertThat(configuredTarget).isInstanceOf(RuleConfiguredTarget.class);
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- assertThat(actions).hasSize(1);
-
- ActionAnalysisMetadata action = Iterables.getOnlyElement(actions);
- assertThat(action).isInstanceOf(SymlinkAction.class);
- SymlinkAction symlinkAction = (SymlinkAction) action;
- assertThat(symlinkAction.executeUnconditionally()).isTrue();
- assertThat(symlinkAction.getInputPath())
- .isEqualTo(PathFragment.create("/workspace/build_config/input.txt"));
- assertThat(symlinkAction.getPrimaryOutput().getExecPathString())
- .isEqualTo("build_config/input.txt");
-
- NinjaGraphProvider provider = configuredTarget.getProvider(NinjaGraphProvider.class);
- assertThat(provider).isNotNull();
- assertThat(provider.getOutputRoot()).isEqualTo(PathFragment.create("build_config"));
- assertThat(provider.getWorkingDirectory()).isEqualTo(PathFragment.EMPTY_FRAGMENT);
- assertThat(provider.getPhonyTargetsMap()).isEmpty();
- assertThat(provider.getTargetsMap()).hasSize(1);
-
- NinjaTarget target = Iterables.getOnlyElement(provider.getTargetsMap().values());
- assertThat(target.getRuleName()).isEqualTo("echo");
- assertThat(target.getAllInputs())
- .containsExactly(PathFragment.create("build_config/input.txt"));
- assertThat(target.getAllOutputs())
- .containsExactly(PathFragment.create("build_config/hello.txt"));
- }
-
- @Test
- public void testNinjaGraphRuleWithOutputRootInputDirs() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- scratch.file("build_config/input.txt", "One");
- scratch.file("build_config/deps/source_files/input2.txt", "Two");
- scratch.file(
- "build_config/build.ninja",
- "rule echo",
- " command = echo \"$$(cat ${in})!\" > ${out}",
- "build build_config/output.txt: echo build_config/input.txt "
- + "build_config/deps/source_files/input2.txt");
-
- // Working directory is workspace root.
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "graph",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " main = 'build_config/build.ninja',",
- " output_root_input_dirs = ['deps/source_files'],",
- " output_root_inputs = ['input.txt'])");
- assertThat(configuredTarget).isInstanceOf(RuleConfiguredTarget.class);
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- assertThat(actions).hasSize(2);
- assertThat(
- actions.stream()
- .map(a -> ((SymlinkAction) a).getInputPath())
- .collect(toImmutableList()))
- .containsExactly(
- PathFragment.create("/workspace/build_config/input.txt"),
- PathFragment.create("/workspace/build_config/deps/source_files/input2.txt"));
-
- NinjaGraphProvider provider = configuredTarget.getProvider(NinjaGraphProvider.class);
- assertThat(provider).isNotNull();
- assertThat(provider.getOutputRoot()).isEqualTo(PathFragment.create("build_config"));
- assertThat(provider.getWorkingDirectory()).isEqualTo(PathFragment.EMPTY_FRAGMENT);
- assertThat(provider.getPhonyTargetsMap()).isEmpty();
-
- assertThat(provider.getOutputRootInputsSymlinks()).hasSize(2);
- assertThat(provider.getOutputRootInputsSymlinks())
- .containsExactly(
- PathFragment.create("build_config/input.txt"),
- PathFragment.create("build_config/deps/source_files/input2.txt"));
-
- assertThat(provider.getTargetsMap()).hasSize(1);
- NinjaTarget target = Iterables.getOnlyElement(provider.getTargetsMap().values());
- assertThat(target.getRuleName()).isEqualTo("echo");
- assertThat(target.getAllInputs())
- .containsExactly(
- PathFragment.create("build_config/input.txt"),
- PathFragment.create("build_config/deps/source_files/input2.txt"));
- assertThat(target.getAllOutputs())
- .containsExactly(PathFragment.create("build_config/output.txt"));
- }
-
- @Test
- public void testNinjaGraphRuleWithPhonyTarget() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- // We do not have to have the real files in place, the rule only reads
- // the contents of Ninja files.
- scratch.file(
- "build_config/build.ninja",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- "build hello.txt: echo input.txt",
- "build alias: phony hello.txt");
-
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "graph",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " working_directory = 'build_config',",
- " main = 'build_config/build.ninja',",
- " output_root_inputs = ['input.txt'])");
- assertThat(configuredTarget).isInstanceOf(RuleConfiguredTarget.class);
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- assertThat(configuredTarget).isInstanceOf(RuleConfiguredTarget.class);
-
- assertThat(actions).hasSize(1);
- ActionAnalysisMetadata action = Iterables.getOnlyElement(actions);
- assertThat(action).isInstanceOf(SymlinkAction.class);
- SymlinkAction symlinkAction = (SymlinkAction) action;
- assertThat(symlinkAction.executeUnconditionally()).isTrue();
- assertThat(symlinkAction.getInputPath())
- .isEqualTo(PathFragment.create("/workspace/build_config/input.txt"));
- assertThat(symlinkAction.getPrimaryOutput().getExecPathString())
- .isEqualTo("build_config/input.txt");
-
- NinjaGraphProvider provider = configuredTarget.getProvider(NinjaGraphProvider.class);
- assertThat(provider).isNotNull();
- assertThat(provider.getOutputRoot()).isEqualTo(PathFragment.create("build_config"));
- assertThat(provider.getWorkingDirectory()).isEqualTo(PathFragment.create("build_config"));
- assertThat(provider.getTargetsMap()).hasSize(1);
-
- NinjaTarget target = Iterables.getOnlyElement(provider.getTargetsMap().values());
- assertThat(target.getRuleName()).isEqualTo("echo");
- assertThat(target.getAllInputs()).containsExactly(PathFragment.create("input.txt"));
- assertThat(target.getAllOutputs()).containsExactly(PathFragment.create("hello.txt"));
-
- PathFragment alias = PathFragment.create("alias");
- assertThat(provider.getPhonyTargetsMap().keySet()).containsExactly(alias);
- PhonyTarget phonyTarget = provider.getPhonyTargetsMap().get(alias);
- assertThat(phonyTarget.isAlwaysDirty()).isFalse();
- assertThat(phonyTarget.getPhonyNames()).isEmpty();
- assertThat(phonyTarget.getDirectExplicitInputs())
- .containsExactly(PathFragment.create("hello.txt"));
- }
-
- @Test
- public void testNinjaGraphRuleWithPhonyTree() throws Exception {
- rewriteWorkspace(
- "workspace(name = 'test')", "toplevel_output_directories(paths = ['build_config'])");
-
- // We do not have to have the real files in place, the rule only reads
- // the contents of Ninja files.
- scratch.file(
- "build_config/build.ninja",
- "rule cat",
- " command = cat ${in} > ${out}",
- "rule echo",
- " command = echo \"Hello $$(cat ${in} | tr '\\r\\n' ' ')!\" > ${out}",
- "build a: cat a.txt",
- "build b: cat b.txt",
- "build c: cat c.txt",
- "build d: cat d.txt",
- // e should be executed unconditionally as it depends on always-dirty phony action
- "build e: cat e.txt always_dirty",
- "build always_dirty: phony",
- "build group1: phony a b c",
- "build group2: phony d e",
- "build inputs_alias: phony group1 group2",
- "build hello.txt: echo inputs_alias",
- "build alias: phony hello.txt");
-
- ConfiguredTarget configuredTarget =
- scratchConfiguredTarget(
- "",
- "graph",
- "ninja_graph(name = 'graph', output_root = 'build_config',",
- " working_directory = 'build_config',",
- " main = 'build_config/build.ninja',",
- " output_root_inputs = ['a.txt', 'b.txt', 'c.txt', 'd.txt', 'e.txt'])");
- assertThat(configuredTarget).isInstanceOf(RuleConfiguredTarget.class);
- RuleConfiguredTarget ninjaConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
- ImmutableList<ActionAnalysisMetadata> actions = ninjaConfiguredTarget.getActions();
- assertThat(actions).hasSize(5);
- List<String> outputs = Lists.newArrayList();
- actions.forEach(a -> outputs.add(Iterables.getOnlyElement(a.getOutputs()).getExecPathString()));
- assertThat(outputs)
- .containsExactlyElementsIn(
- new String[] {
- "build_config/a.txt",
- "build_config/b.txt",
- "build_config/c.txt",
- "build_config/d.txt",
- "build_config/e.txt"
- });
-
- for (ActionAnalysisMetadata action : actions) {
- assertThat(action).isInstanceOf(SymlinkAction.class);
- SymlinkAction symlinkAction = (SymlinkAction) action;
- assertThat(symlinkAction.executeUnconditionally()).isTrue();
- assertThat(symlinkAction.getInputPath().getParentDirectory())
- .isEqualTo(PathFragment.create("/workspace/build_config"));
- assertThat(symlinkAction.getInputPath().getFileExtension()).isEqualTo("txt");
- PathFragment execRootPath = symlinkAction.getPrimaryOutput().getExecPath();
- assertThat(execRootPath.getParentDirectory()).isEqualTo(PathFragment.create("build_config"));
- assertThat(execRootPath.getFileExtension()).isEqualTo("txt");
- }
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerStepTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerStepTest.java
deleted file mode 100644
index 6d8df0b..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerStepTest.java
+++ /dev/null
@@ -1,203 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja;
-
-import static com.google.common.truth.Truth.assertThat;
-
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexerStep;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.function.Consumer;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexerStep}. */
-@RunWith(JUnit4.class)
-public class NinjaLexerStepTest {
- @Test
- public void testSkipSpaces() {
- NinjaLexerStep step = step(" ");
- assertThat(step.canAdvance()).isTrue();
- assertThat(step.startByte()).isEqualTo(' ');
- step.skipSpaces();
- assertThat(step.getEnd()).isEqualTo(2);
- assertThat(step.getBytes()).isEqualTo(" ".getBytes(StandardCharsets.ISO_8859_1));
- assertThat(step.canAdvance()).isFalse();
- }
-
- @Test
- public void testSkipComment() {
- doTest("# skfdskl werj wior982438923u rekfjg wef w $ $: : |", NinjaLexerStep::skipComment);
- doTest("#", NinjaLexerStep::skipComment);
- doTest("# 123\n", NinjaLexerStep::skipComment, "# 123", true);
- doTest("# 123\u0000118", NinjaLexerStep::skipComment, "# 123", false);
- }
-
- @Test
- public void testTrySkipEscapedNewline() {
- doTest("$\n", step -> assertThat(step.trySkipEscapedNewline()).isTrue());
- doTest("$\n\n", step -> assertThat(step.trySkipEscapedNewline()).isTrue(), "$\n", true);
- doTest("$\r\n", step -> assertThat(step.trySkipEscapedNewline()).isTrue());
- doTest("$\r\na", step -> assertThat(step.trySkipEscapedNewline()).isTrue(), "$\r\n", true);
- }
-
- @Test
- public void testProcessLineFeedNewLine() {
- doTest("\r\n", NinjaLexerStep::processLineFeedNewLine);
- doTestError(
- "\ra",
- NinjaLexerStep::processLineFeedNewLine,
- "\ra",
- false,
- "Wrong newline separators: \\r should be followed by \\n.");
- }
-
- @Test
- public void testTryReadVariableInBrackets() {
- doTest("${abc}", step -> assertThat(step.tryReadVariableInBrackets()).isTrue());
- doTest("$abc", step -> assertThat(step.tryReadVariableInBrackets()).isFalse(), "", true);
-
- doTest("${ abc }", step -> assertThat(step.tryReadVariableInBrackets()).isTrue());
- doTest(
- "${abc.xyz-1_2}cde",
- step -> assertThat(step.tryReadVariableInBrackets()).isTrue(),
- "${abc.xyz-1_2}",
- true);
- doTestError(
- "${abc",
- step -> assertThat(step.tryReadVariableInBrackets()).isTrue(),
- "${abc",
- false,
- "Variable end symbol '}' expected.");
- doTestError(
- "${abc\n",
- step -> assertThat(step.tryReadVariableInBrackets()).isTrue(),
- "${abc\n",
- false,
- "Variable end symbol '}' expected.");
- doTestError(
- "${}",
- step -> assertThat(step.tryReadVariableInBrackets()).isTrue(),
- "${}",
- false,
- "Variable identifier expected.");
- doTestError(
- "${^}",
- step -> assertThat(step.tryReadVariableInBrackets()).isTrue(),
- "${^",
- true,
- "Variable identifier expected.");
- }
-
- @Test
- public void testTryReadSimpleVariable() {
- doTest("$abc", step -> assertThat(step.tryReadSimpleVariable()).isTrue());
- doTest("$a-b_c", step -> assertThat(step.tryReadSimpleVariable()).isTrue());
- doTest("$.", step -> assertThat(step.tryReadSimpleVariable()).isFalse(), "", true);
- doTest("$abc.cde", step -> assertThat(step.tryReadSimpleVariable()).isTrue(), "$abc", true);
- }
-
- @Test
- public void testTryReadEscapedLiteral() {
- doTest("$:", step -> assertThat(step.tryReadEscapedLiteral()).isTrue(), "$:", false);
- doTest("$$", step -> assertThat(step.tryReadEscapedLiteral()).isTrue(), "$$", false);
- doTest("$ ", step -> assertThat(step.tryReadEscapedLiteral()).isTrue(), "$ ", false);
-
- doTest("$:a", step -> assertThat(step.tryReadEscapedLiteral()).isTrue(), "$:", true);
- doTest("$$$", step -> assertThat(step.tryReadEscapedLiteral()).isTrue(), "$$", true);
- doTest("$ ", step -> assertThat(step.tryReadEscapedLiteral()).isTrue(), "$ ", true);
-
- doTest("$a", step -> assertThat(step.tryReadEscapedLiteral()).isFalse(), "", true);
- }
-
- @Test
- public void testTryReadIdentifier() {
- doTest("abc_d-18", NinjaLexerStep::tryReadIdentifier);
- doTest("abc_d-18.ccc", NinjaLexerStep::tryReadIdentifier);
- doTest("abc_d-18.ccc=", NinjaLexerStep::tryReadIdentifier, "abc_d-18.ccc", true);
- // Have a longer text to demonstrate the error output.
- doTestError(
- "^abc Bazel only rebuilds what is necessary. "
- + "With advanced local and distributed caching, optimized dependency analysis "
- + "and parallel execution, you get fast and incremental builds.",
- NinjaLexerStep::tryReadIdentifier,
- "^",
- true,
- "Symbol '^' is not allowed in the identifier, the text fragment with the symbol:\n"
- + "^abc Bazel only rebuilds what is necessary. With advanced local and distributed"
- + " caching, optimized dependency analysis and parallel execution,"
- + " you get fast and incremental builds.\n");
- }
-
- @Test
- public void testReadPath() {
- doTest(
- "this/is/the/relative/path.txt",
- NinjaLexerStep::readPath,
- "this/is/the/relative/path.txt",
- false);
- doTest(
- "relative/text#.properties", NinjaLexerStep::readPath, "relative/text#.properties", false);
- }
-
- @Test
- public void testTryReadDoublePipe() {
- doTest("||", NinjaLexerStep::tryReadDoublePipe);
- }
-
- @Test
- public void testReadText() {
- doTest("text$\npart", NinjaLexerStep::readText, "text", true);
- doTest("one word", NinjaLexerStep::readText, "one", true);
- }
-
- private static void doTest(String text, Consumer<NinjaLexerStep> callback) {
- doTest(text, callback, text, false);
- }
-
- private static void doTest(
- String text, Consumer<NinjaLexerStep> callback, String expected, boolean haveMore) {
- NinjaLexerStep step = step(text);
- assertThat(step.canAdvance()).isTrue();
- callback.accept(step);
- assertThat(step.getError()).isNull();
- if (!expected.isEmpty()) {
- assertThat(step.getBytes()).isEqualTo(expected.getBytes(StandardCharsets.ISO_8859_1));
- }
- assertThat(step.canAdvance()).isEqualTo(haveMore);
- }
-
- private static void doTestError(
- String text,
- Consumer<NinjaLexerStep> callback,
- String expected,
- boolean haveMore,
- String errorText) {
- NinjaLexerStep step = step(text);
- assertThat(step.canAdvance()).isTrue();
- callback.accept(step);
- assertThat(step.getError()).isEqualTo(errorText);
- assertThat(step.getBytes()).isEqualTo(expected.getBytes(StandardCharsets.ISO_8859_1));
- assertThat(step.getFragment().length() > step.getEnd()).isEqualTo(haveMore);
- }
-
- private static NinjaLexerStep step(String text) {
- ByteBuffer bb = ByteBuffer.wrap(text.getBytes(StandardCharsets.ISO_8859_1));
- return new NinjaLexerStep(new FileFragment(bb, 0, 0, bb.limit()), 0);
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerTest.java
deleted file mode 100644
index c7fea79..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaLexerTest.java
+++ /dev/null
@@ -1,238 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja;
-
-import static com.google.common.truth.Truth.assertThat;
-
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer;
-import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer.TextKind;
-import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaToken;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import javax.annotation.Nullable;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer}. */
-@RunWith(JUnit4.class)
-public class NinjaLexerTest {
-
- @Test
- public void testReadIdentifiersAndVariables() {
- String text = "abc efg $fg ${ghf}\ntext one ${ more.1-d_f } $abc.def";
- NinjaLexer lexer = createLexer(text);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "abc");
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "efg");
- assertTokenBytes(lexer, NinjaToken.VARIABLE, "$fg");
- assertTokenBytes(lexer, NinjaToken.VARIABLE, "${ghf}");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "text");
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "one");
- assertTokenBytes(lexer, NinjaToken.VARIABLE, "${ more.1-d_f }");
- assertTokenBytes(lexer, NinjaToken.VARIABLE, "$abc");
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, ".def");
- }
-
- @Test
- public void testNewlines() {
- String text = "a\nb $\nnot-newline$$\nnewline\n\nand\r\none";
- NinjaLexer lexer = createLexer(text);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "a");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "b");
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "not-newline");
- assertTokenBytes(lexer, NinjaToken.ESCAPED_TEXT, "$$");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "newline");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "and");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "one");
- assertThat(lexer.hasNextToken()).isFalse();
- }
-
- @Test
- public void testTabsAreAllowed() {
- String text = "abc\n\tcde";
- NinjaLexer lexer = createLexer(text);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "abc");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- assertTokenBytes(lexer, NinjaToken.INDENT, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "cde");
- }
-
- @Test
- public void testDisallowedSymbols() {
- assertError(
- createLexer("^"),
- "Symbol '^' is not allowed in the identifier, the text fragment with the symbol:\n^\n",
- "^");
- }
-
- @Test
- public void testComments() {
- String text =
- "abc#immediately after\n#Start of the line $ not escaped in comment $"
- + "\nNot-comment# Finishing : = $ | ||";
- NinjaLexer lexer = createLexer(text);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "abc");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "Not-comment");
- }
-
- @Test
- public void testBadEscape() {
- NinjaLexer lexer = createLexer("abc\nbad $");
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "abc");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "bad");
- assertError(lexer, "Bad $-escape (literal $ must be written as $$)", "$");
-
- NinjaLexer lexer2 = createLexer("$$$");
- assertTokenBytes(lexer2, NinjaToken.ESCAPED_TEXT, "$$");
- assertError(lexer2, "Bad $-escape (literal $ must be written as $$)", "$");
- }
-
- @Test
- public void testBadVariable() {
- assertError(createLexer("${abc"), "Variable end symbol '}' expected.", "${abc");
- assertError(createLexer("${abc "), "Variable end symbol '}' expected.", "${abc ");
- assertError(createLexer("${}"), "Variable identifier expected.", "${}");
- assertError(createLexer("${abc&}"), "Variable end symbol '}' expected.", "${abc&");
- }
-
- @Test
- public void testKeywords() {
- assertTokenBytes(createLexer("build"), NinjaToken.BUILD, null);
- assertTokenBytes(createLexer("rule"), NinjaToken.RULE, null);
- assertTokenBytes(createLexer("default "), NinjaToken.DEFAULT, null);
- assertTokenBytes(createLexer("include"), NinjaToken.INCLUDE, null);
- assertTokenBytes(createLexer("subninja\n"), NinjaToken.SUBNINJA, null);
- assertTokenBytes(createLexer("pool "), NinjaToken.POOL, null);
- }
-
- @Test
- public void testIndent() {
- NinjaLexer lexer = createLexer(" a\nb\n c d e\n ");
- // We want to know if there was a starting INDENT
- // (though we suppose to start with a line without INDENT)
- assertTokenBytes(lexer, NinjaToken.INDENT, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "a");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "b");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- assertTokenBytes(lexer, NinjaToken.INDENT, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "c");
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "d");
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "e");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- assertTokenBytes(lexer, NinjaToken.INDENT, null);
- }
-
- @Test
- public void testReadTextFragment() {
- NinjaLexer lexer = createLexer("my.var=Any text ^&%=@&!*: $:symbols$\n aa\nmy.var2");
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "my.var");
- assertTokenBytes(lexer, NinjaToken.EQUALS, null);
-
- lexer.setExpectedTextKind(TextKind.TEXT);
- assertTokenBytes(lexer, NinjaToken.TEXT, "Any");
- assertTokenBytes(lexer, NinjaToken.TEXT, "text");
- assertTokenBytes(lexer, NinjaToken.TEXT, "^&%=@&!*");
- assertTokenBytes(lexer, NinjaToken.COLON, null);
- assertTokenBytes(lexer, NinjaToken.ESCAPED_TEXT, "$:");
- assertTokenBytes(lexer, NinjaToken.TEXT, "symbols");
- assertTokenBytes(lexer, NinjaToken.TEXT, "aa");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "my.var2");
- }
-
- @Test
- public void testUndo() {
- NinjaLexer lexer = createLexer("my.var=Any\n");
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "my.var");
- assertTokenBytes(lexer, NinjaToken.EQUALS, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "Any");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
-
- lexer.undo();
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- lexer.undo();
- lexer.undo();
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "Any");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- lexer.undo();
- lexer.undo();
- lexer.undo();
- assertTokenBytes(lexer, NinjaToken.EQUALS, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "Any");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- lexer.undo();
- lexer.undo();
- lexer.undo();
- lexer.undo();
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "my.var");
- lexer.undo();
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "my.var");
- assertTokenBytes(lexer, NinjaToken.EQUALS, null);
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, "Any");
- assertTokenBytes(lexer, NinjaToken.NEWLINE, null);
- }
-
- @Test
- public void testSpecialSymbols() {
- NinjaLexer lexer = createLexer("| || : = ");
- assertTokenBytes(lexer, NinjaToken.PIPE, null);
- assertTokenBytes(lexer, NinjaToken.PIPE2, null);
- assertTokenBytes(lexer, NinjaToken.COLON, null);
- assertTokenBytes(lexer, NinjaToken.EQUALS, null);
- assertTokenBytes(lexer, NinjaToken.TEXT, " ");
- assertThat(lexer.hasNextToken()).isFalse();
- }
-
- @Test
- public void testZeroByte() {
- byte[] bytes = {'a', 0, 'b'};
- NinjaLexer lexer = new NinjaLexer(new FileFragment(ByteBuffer.wrap(bytes), 0, 0, bytes.length));
- assertTokenBytes(lexer, NinjaToken.IDENTIFIER, null);
- assertThat(lexer.hasNextToken()).isFalse();
- }
-
- private static void assertError(NinjaLexer lexer, String errorText, String errorHolder) {
- assertThat(lexer.hasNextToken()).isTrue();
- assertThat(lexer.nextToken()).isEqualTo(NinjaToken.ERROR);
- assertThat(lexer.getError()).isEqualTo(errorText);
- assertThat(lexer.getTokenBytes()).isEqualTo(errorHolder.getBytes(StandardCharsets.ISO_8859_1));
- assertThat(lexer.hasNextToken()).isFalse();
- }
-
- private static void assertTokenBytes(NinjaLexer lexer, NinjaToken token, @Nullable String text) {
- assertThat(lexer.hasNextToken()).isTrue();
- assertThat(lexer.nextToken()).isEqualTo(token);
- if (text != null) {
- assertThat(lexer.getTokenBytes()).isEqualTo(text.getBytes(StandardCharsets.ISO_8859_1));
- }
- }
-
- private static NinjaLexer createLexer(String text) {
- ByteBuffer buffer = ByteBuffer.wrap(text.getBytes(StandardCharsets.ISO_8859_1));
- return new NinjaLexer(new FileFragment(buffer, 0, 0, buffer.limit()));
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaParserStepTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaParserStepTest.java
deleted file mode 100644
index d60e812..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaParserStepTest.java
+++ /dev/null
@@ -1,613 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja;
-
-import static com.google.common.truth.Truth.assertThat;
-import static org.junit.Assert.assertThrows;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.Maps;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaFileParseResult;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaParserStep;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaPool;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaRule;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaRuleVariable;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaScope;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaVariableValue;
-import com.google.devtools.build.lib.concurrent.BlazeInterners;
-import com.google.devtools.build.lib.util.Pair;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.function.Function;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link NinjaParserStep}. */
-@RunWith(JUnit4.class)
-public class NinjaParserStepTest {
- private static final int LINE_NUM_AFTER_RULE_DEFS = 5;
-
- @Test
- public void testSimpleVariable() throws Exception {
- doTestSimpleVariable("a=b", "a", "b");
- doTestSimpleVariable("a=b\nc", "a", "b");
- doTestSimpleVariable("a=b # comment", "a", "b");
- doTestSimpleVariable("a.b.c = some long: value", "a.b.c", "some long: value");
- doTestSimpleVariable("a_11_24-rt.15= ^&%=#@", "a_11_24-rt.15", "^&%=#@");
- doTestSimpleVariable("a= $\n b", "a", "b");
- }
-
- @Test
- public void testTrailingWhitespaceIsSaved() throws Exception {
- doTestSimpleVariable("a = b ", "a", "b ");
- }
-
- @Test
- public void testTrailingWhitespaceBeforeNewlineIsSaved() throws Exception {
- doTestSimpleVariable("a = b \n", "a", "b ");
- }
-
- @Test
- public void testTrailingWhitespaceBeforeCarriageReturnIsSaved() throws Exception {
- doTestSimpleVariable("a = b \r", "a", "b ");
- }
-
- @Test
- public void testTrailingWhitespaceBeforeNewlineCarriageReturnIsSaved() throws Exception {
- doTestSimpleVariable("a = b \n\r", "a", "b ");
- }
-
- @Test
- public void testTrailingWhitespaceBeforeCarriageReturnNewlineIsSaved() throws Exception {
- doTestSimpleVariable("a = b \r\n", "a", "b ");
- }
-
- @Test
- public void testVariableParsingException() {
- doTestVariableParsingException(" ", "Expected identifier, but got indent in fragment:\n \n");
- doTestVariableParsingException("a", "Expected = after 'a' in fragment:\na\n");
- doTestVariableParsingException(
- "^a=",
- "Expected identifier, but got error: 'Symbol '^' is not allowed in the identifier, "
- + "the text fragment with the symbol:\n^a=\n' in fragment:\n^a=\n");
- }
-
- private static void doTestVariableParsingException(String text, String message) {
- GenericParsingException exception =
- assertThrows(GenericParsingException.class, () -> createParser(text).parseVariable());
- assertThat(exception).hasMessageThat().isEqualTo(message);
- }
-
- @Test
- public void testNoValue() throws Exception {
- doTestNoValue("a=");
- doTestNoValue("a =");
- doTestNoValue("a =");
- doTestNoValue("a=\u000018");
- doTestNoValue("a = ");
- doTestNoValue("a = ");
- doTestNoValue("a = ");
- doTestNoValue("a =\nm");
- doTestNoValue("a = # 123");
- }
-
- @Test
- public void testWithVariablesInValue() throws Exception {
- doTestWithVariablesInValue("a=$a $b", "a", "${a} ${b}", ImmutableSortedSet.of("a", "b"));
- doTestWithVariablesInValue("a=a_$b_c", "a", "a_${b_c}", ImmutableSortedSet.of("b_c"));
- doTestWithVariablesInValue("a=$b a c", "a", "${b} a c", ImmutableSortedSet.of("b"));
- doTestWithVariablesInValue("a=a_$b c", "a", "a_${b} c", ImmutableSortedSet.of("b"));
- doTestWithVariablesInValue("a=a_${b.d}c", "a", "a_${b.d}c", ImmutableSortedSet.of("b.d"));
- doTestWithVariablesInValue(
- "e=a$b*c${ d }*18", "e", "a${b}*c${d}*18", ImmutableSortedSet.of("b", "d"));
- doTestWithVariablesInValue("e=a$b*${ b }", "e", "a${b}*${b}", ImmutableSortedSet.of("b"));
- }
-
- @Test
- public void testNormalizeVariableName() {
- assertThat(NinjaParserStep.normalizeVariableName("$a")).isEqualTo("a");
- assertThat(NinjaParserStep.normalizeVariableName("$a-b-c")).isEqualTo("a-b-c");
- assertThat(NinjaParserStep.normalizeVariableName("${abc_de-7}")).isEqualTo("abc_de-7");
- assertThat(NinjaParserStep.normalizeVariableName("${ a1.5}")).isEqualTo("a1.5");
- assertThat(NinjaParserStep.normalizeVariableName("${a1.5 }")).isEqualTo("a1.5");
- }
-
- @Test
- public void testInclude() throws Exception {
- NinjaVariableValue value1 = createParser("include x/multi words/z").parseIncludeStatement();
- assertThat(value1.getRawText()).isEqualTo("x/multi words/z");
-
- NinjaVariableValue value2 = createParser("subninja ${x}.ninja").parseSubNinjaStatement();
- assertThat(value2.getRawText()).isEqualTo("${x}.ninja");
- MockValueExpander expander = new MockValueExpander("###");
- assertThat(value2.getExpandedValue(expander)).isEqualTo("###x.ninja");
- assertThat(expander.getRequestedVariables()).containsExactly("x");
- }
-
- @Test
- public void testIncludeErrors() {
- GenericParsingException exception1 =
- assertThrows(
- GenericParsingException.class,
- () -> createParser("include x $").parseIncludeStatement());
- assertThat(exception1)
- .hasMessageThat()
- .isEqualTo(
- "Expected newline, but got error: "
- + "'Bad $-escape (literal $ must be written as $$)' in fragment:\ninclude x $\n");
-
- GenericParsingException exception2 =
- assertThrows(
- GenericParsingException.class, () -> createParser("include").parseIncludeStatement());
- assertThat(exception2).hasMessageThat().isEqualTo("include statement has no path.");
-
- GenericParsingException exception3 =
- assertThrows(
- GenericParsingException.class,
- () -> createParser("subninja \nm").parseSubNinjaStatement());
- assertThat(exception3).hasMessageThat().isEqualTo("subninja statement has no path.");
- }
-
- @Test
- public void testNinjaRule() throws Exception {
- // Additionally test the situation when we get more line separators in the end.
- NinjaParserStep parser =
- createParser(
- "rule testRule \n"
- + " command = executable --flag $TARGET $out && $POST_BUILD\n"
- + " description = Test rule for $TARGET\n"
- + " rspfile = $TARGET.in\n"
- + " deps = ${abc} $\n"
- + " ${cde}\n\n\n");
- NinjaRule ninjaRule = parser.parseNinjaRule();
- ImmutableSortedMap<NinjaRuleVariable, NinjaVariableValue> variables = ninjaRule.getVariables();
- assertThat(variables.keySet())
- .containsExactly(
- NinjaRuleVariable.COMMAND,
- NinjaRuleVariable.DESCRIPTION,
- NinjaRuleVariable.RSPFILE,
- NinjaRuleVariable.DEPS);
- assertThat(ninjaRule.getName()).isEqualTo("testRule");
- assertThat(variables.get(NinjaRuleVariable.DEPS).getRawText()).isEqualTo("${abc} $\n ${cde}");
- MockValueExpander expander = new MockValueExpander("###");
- assertThat(variables.get(NinjaRuleVariable.DEPS).getExpandedValue(expander))
- .isEqualTo("###abc $\n ###cde");
- assertThat(variables.get(NinjaRuleVariable.DESCRIPTION).getRawText())
- .isEqualTo("Test rule for ${TARGET}");
- assertThat(expander.getRequestedVariables()).containsExactly("abc", "cde");
- }
-
- @Test
- public void testNinjaRuleWithHash() throws Exception {
- // Additionally test the situation when we get more line separators in the end.
- NinjaParserStep parser =
- createParser(
- "rule testRule \n"
- + " command = executable --flag $TARGET $out && sed -e 's/#.*$$//' -e '/^$$/d'\n"
- + " description = Test rule for $TARGET");
- NinjaRule ninjaRule = parser.parseNinjaRule();
- assertThat(ninjaRule.getVariables().get(NinjaRuleVariable.COMMAND).getRawText())
- // Variables are wrapped with {} by print function, $$ escape sequence is unescaped.
- .isEqualTo("executable --flag ${TARGET} ${out} && sed -e 's/#.*$//' -e '/^$/d'");
- }
-
- @Test
- public void testNinjaRuleWithDollarSign() throws Exception {
- NinjaParserStep parser =
- createParser(
- "rule testRule \n"
- + " command = something && $\n"
- + " something_else\n"
- + " description = Test $\n"
- + " rule");
- NinjaRule ninjaRule = parser.parseNinjaRule();
- // Ensure that the dollar sign doesn't exist in the parsed command variable value.
- // Deliberately omit them because it could cause problems in multiline shell commands.
- assertThat(ninjaRule.getVariables().get(NinjaRuleVariable.COMMAND).getRawText())
- .isEqualTo("something && \n something_else");
- assertThat(ninjaRule.getVariables().get(NinjaRuleVariable.DESCRIPTION).getRawText())
- .isEqualTo("Test \n rule");
- }
-
- @Test
- public void testNinjaRuleWithStickyDollarSign() throws Exception {
- NinjaParserStep parser =
- createParser(
- "rule testRule \n"
- + " command = something &&$\n"
- + " something_else\n"
- + " description = Test$\n"
- + " rule");
- NinjaRule ninjaRule = parser.parseNinjaRule();
- // Ensure that the dollar sign doesn't exist in the parsed command variable value.
- // Deliberately omit them because it could cause problems in multiline shell commands.
- assertThat(ninjaRule.getVariables().get(NinjaRuleVariable.COMMAND).getRawText())
- .isEqualTo("something &&\n something_else");
- assertThat(ninjaRule.getVariables().get(NinjaRuleVariable.DESCRIPTION).getRawText())
- .isEqualTo("Test\n rule");
- }
-
- @Test
- public void testVariableWithoutValue() throws Exception {
- NinjaParserStep parser =
- createParser(
- "rule testRule \n"
- + " command = executable --flag $TARGET $out && $POST_BUILD\n"
- + " description =\n");
- NinjaRule ninjaRule = parser.parseNinjaRule();
- ImmutableSortedMap<NinjaRuleVariable, NinjaVariableValue> variables = ninjaRule.getVariables();
- assertThat(variables.keySet())
- .containsExactly(NinjaRuleVariable.COMMAND, NinjaRuleVariable.DESCRIPTION);
- assertThat(ninjaRule.getName()).isEqualTo("testRule");
- assertThat(variables.get(NinjaRuleVariable.DESCRIPTION).getRawText()).isEmpty();
- }
-
- @Test
- public void testParsePoolInRule() throws Exception {
- NinjaParserStep parser = createParser("rule testRule \n" + " pool = some_pool\n");
- NinjaRule ninjaRule = parser.parseNinjaRule();
- ImmutableSortedMap<NinjaRuleVariable, NinjaVariableValue> variables = ninjaRule.getVariables();
- assertThat(variables.keySet()).containsExactly(NinjaRuleVariable.POOL);
- assertThat(ninjaRule.getName()).isEqualTo("testRule");
- assertThat(variables.get(NinjaRuleVariable.POOL).getRawText()).isEqualTo("some_pool");
- }
-
- @Test
- public void testParsePoolDeclaration() throws Exception {
- NinjaParserStep parser = createParser("pool link_pool\n" + " depth = 4\n");
- NinjaPool ninjaPool = parser.parseNinjaPool();
- assertThat(ninjaPool.getName()).isEqualTo("link_pool");
- assertThat(ninjaPool.getDepth()).isEqualTo(4);
- }
-
- @Test
- public void testParsePoolDeclaration_withInvalidDepth() throws Exception {
- NinjaParserStep parser = createParser("pool link_pool\n" + " depth = NaN\n");
- assertThrows(
- "Expected an integer for the 'depth' value, but got 'NaN'.",
- GenericParsingException.class,
- parser::parseNinjaPool);
- }
-
- @Test
- public void testParseDescriptionInRule() throws Exception {
- NinjaParserStep parser = createParser("rule testRule \n" + " description = foobar $out\n");
- NinjaRule ninjaRule = parser.parseNinjaRule();
- ImmutableSortedMap<NinjaRuleVariable, NinjaVariableValue> variables = ninjaRule.getVariables();
- assertThat(variables.keySet()).containsExactly(NinjaRuleVariable.DESCRIPTION);
- assertThat(ninjaRule.getName()).isEqualTo("testRule");
- assertThat(ninjaRule.getDescription()).isEqualTo("foobar ${out}");
- }
-
- @Test
- public void testNoDescriptionInRule_isEmptyString() throws Exception {
- NinjaParserStep parser = createParser("rule testRule \n" + " command = foobar\n");
- NinjaRule ninjaRule = parser.parseNinjaRule();
- ImmutableSortedMap<NinjaRuleVariable, NinjaVariableValue> variables = ninjaRule.getVariables();
- assertThat(variables.keySet()).containsExactly(NinjaRuleVariable.COMMAND);
- assertThat(ninjaRule.getName()).isEqualTo("testRule");
- assertThat(ninjaRule.getDescription()).isEmpty();
- }
-
- @Test
- public void testNinjaRuleParsingException() {
- doTestNinjaRuleParsingException(
- "rule testRule extra-word\n",
- String.join(
- "\n",
- "Expected newline, but got identifier in fragment:",
- "rule testRule extra-word",
- "",
- ""));
- doTestNinjaRuleParsingException(
- "rule testRule\ncommand =",
- String.join(
- "\n",
- "Expected indent, but got identifier in fragment:",
- "rule testRule",
- "command =",
- ""));
- doTestNinjaRuleParsingException(
- "rule testRule\n ^custom = a",
- String.join(
- "\n",
- "Expected identifier, but got error: 'Symbol '^' is not allowed in the identifier, "
- + "the text fragment with the symbol:",
- "rule testRule",
- " ^custom = a",
- "' in fragment:",
- "rule testRule",
- " ^custom = a",
- ""));
- doTestNinjaRuleParsingException("rule testRule\n custom = a", "Unexpected variable 'custom'");
- }
-
- @Test
- public void testNinjaTargets() throws Exception {
- NinjaScope scope = scopeWithStubRule("command");
-
- // Additionally test the situation when the target does not have the variables section and
- // we get more line separators in the end.
- NinjaTarget target =
- createParser("build output: command input\n\n")
- .parseNinjaTarget(scope, LINE_NUM_AFTER_RULE_DEFS);
- assertThat(target.getRuleName()).isEqualTo("command");
- assertThat(target.getOutputs()).containsExactly(PathFragment.create("output"));
- assertThat(target.getExplicitInputs()).containsExactly(PathFragment.create("input"));
-
- NinjaTarget target1 =
- createParser("build o1 o2 | io1 io2: command i1 i2 | ii1 ii2 || ooi1 ooi2 |@ vi1 vi2")
- .parseNinjaTarget(scope, LINE_NUM_AFTER_RULE_DEFS);
- assertThat(target1.getRuleName()).isEqualTo("command");
- assertThat(target1.getOutputs())
- .containsExactly(PathFragment.create("o1"), PathFragment.create("o2"));
- assertThat(target1.getImplicitOutputs())
- .containsExactly(PathFragment.create("io1"), PathFragment.create("io2"));
- assertThat(target1.getExplicitInputs())
- .containsExactly(PathFragment.create("i1"), PathFragment.create("i2"));
- assertThat(target1.getImplicitInputs())
- .containsExactly(PathFragment.create("ii1"), PathFragment.create("ii2"));
- assertThat(target1.getOrderOnlyInputs())
- .containsExactly(PathFragment.create("ooi1"), PathFragment.create("ooi2"));
- assertThat(target1.getValidationInputs())
- .containsExactly(PathFragment.create("vi1"), PathFragment.create("vi2"));
-
- NinjaTarget target2 =
- createParser("build output: phony").parseNinjaTarget(scope, LINE_NUM_AFTER_RULE_DEFS);
- assertThat(target2.getRuleName()).isEqualTo("phony");
- assertThat(target2.getOutputs()).containsExactly(PathFragment.create("output"));
-
- NinjaTarget target3 =
- createParser("build output: command $\n || order-only-input")
- .parseNinjaTarget(scope, LINE_NUM_AFTER_RULE_DEFS);
- assertThat(target3.getRuleName()).isEqualTo("command");
- assertThat(target3.getOutputs()).containsExactly(PathFragment.create("output"));
- assertThat(target3.getOrderOnlyInputs())
- .containsExactly(PathFragment.create("order-only-input"));
- }
-
- @Test
- public void testNinjaTargetParsingErrors() {
- testNinjaTargetParsingError("build xxx", "Unexpected end of target");
- testNinjaTargetParsingError("build xxx yyy:", "Expected rule name");
- testNinjaTargetParsingError("build xxx || yyy: command", "Unexpected token: PIPE2");
- testNinjaTargetParsingError("build xxx: command :", "Unexpected token: COLON");
- testNinjaTargetParsingError("build xxx: command | || a", "Expected paths sequence");
- testNinjaTargetParsingError("build xxx: command | |@ a", "Expected paths sequence");
- }
-
- @Test
- public void testNinjaTargetsWithVariables() throws Exception {
- NinjaScope scope = new NinjaScope();
-
- // Initialize a rule called "testRule" which takes inputs "dir" and "empty".
- NinjaParserStep parser =
- createParser("rule testRule \n" + " command = executable dir=$dir empty=$empty end");
- NinjaRule ninjaRule = parser.parseNinjaRule();
- scope.setRules(ImmutableSortedMap.of("testRule", ImmutableList.of(Pair.of(0L, ninjaRule))));
-
- NinjaFileParseResult parseResult = new NinjaFileParseResult();
- parseResult.addVariable("output", 1, NinjaVariableValue.createPlainText("out123"));
- parseResult.addVariable("input", 2, NinjaVariableValue.createPlainText("in123"));
- parseResult.expandIntoScope(scope, Maps.newHashMap());
-
- // Variables, defined inside build statement, are used for input and output paths,
- // but not for the values of the other variables.
- // Test it.
- NinjaTarget target =
- createParser(
- "build $output : testRule $input $dir/abcde\n"
- + " dir = def$input\n empty = '$dir'")
- .parseNinjaTarget(scope, LINE_NUM_AFTER_RULE_DEFS);
- assertThat(target.getRuleName()).isEqualTo("testRule");
- assertThat(target.getOutputs()).containsExactly(PathFragment.create("out123"));
- assertThat(target.getExplicitInputs())
- .containsExactly(PathFragment.create("in123"), PathFragment.create("defin123/abcde"));
- assertThat(target.computeRuleVariables().get(NinjaRuleVariable.COMMAND))
- .isEqualTo("executable dir=defin123 empty='' end");
- }
-
- @Test
- public void testPseudoCyclesOfVariables() throws Exception {
- NinjaFileParseResult parseResult = new NinjaFileParseResult();
- parseResult.addVariable(
- "output", 1, NinjaVariableValue.builder().addText("'out'").addVariable("input").build());
- parseResult.addVariable(
- "input", 2, NinjaVariableValue.builder().addText("'in'").addVariable("output").build());
- NinjaScope scope = new NinjaScope();
- parseResult.expandIntoScope(scope, Maps.newHashMap());
- assertThat(scope.findExpandedVariable(3, "input")).isEqualTo("'in''out'");
- assertThat(scope.findExpandedVariable(3, "output")).isEqualTo("'out'");
- }
-
- @Test
- public void testNinjaTargetsPathWithEscapedSpace() throws Exception {
- NinjaScope scope = scopeWithStubRule("command");
-
- NinjaTarget target =
- createParser("build output : command input$ with$ space other")
- .parseNinjaTarget(scope, LINE_NUM_AFTER_RULE_DEFS);
-
- assertThat(target.getRuleName()).isEqualTo("command");
- assertThat(target.getOutputs()).containsExactly(PathFragment.create("output"));
- assertThat(target.getExplicitInputs())
- .containsExactly(PathFragment.create("input with space"), PathFragment.create("other"));
- }
-
- @Test
- public void testNinjaTargetsPathWithEscapedNewline() throws Exception {
- NinjaScope scope = scopeWithStubRule("command");
-
- NinjaTarget target =
- createParser(
- "build $\n" + " output : $\n" + " command input$\n" + " with$\n" + " newline")
- .parseNinjaTarget(scope, LINE_NUM_AFTER_RULE_DEFS);
- assertThat(target.getRuleName()).isEqualTo("command");
- assertThat(target.getOutputs()).containsExactly(PathFragment.create("output"));
- assertThat(target.getExplicitInputs())
- .containsExactly(
- PathFragment.create("input"),
- PathFragment.create("with"),
- PathFragment.create("newline"));
- }
-
- @Test
- public void testNinjaTargetWithScope() throws Exception {
- NinjaScope scope = scopeWithStubRule("command");
-
- NinjaTarget target =
- createParser("build output : command input\n pool = abc\n")
- .parseNinjaTarget(scope, LINE_NUM_AFTER_RULE_DEFS);
-
- assertThat(target.getRuleName()).isEqualTo("command");
- assertThat(target.getOutputs()).containsExactly(PathFragment.create("output"));
- assertThat(target.getExplicitInputs()).containsExactly(PathFragment.create("input"));
- }
-
- @Test
- public void testUndefinedRuleForTarget() throws Exception {
- GenericParsingException exception =
- assertThrows(
- GenericParsingException.class,
- () -> parseNinjaTarget("build output : myUndefinedRule input\n pool = abc\n"));
-
- assertThat(exception).hasMessageThat().contains("could not resolve rule 'myUndefinedRule'");
- }
-
- @Test
- public void testTargetDefinedDescription() throws Exception {
- NinjaScope scope = scopeWithStubRule("testRule");
-
- NinjaTarget target =
- createParser("build output : testRule input\n description = bunny bunny\n")
- .parseNinjaTarget(scope, LINE_NUM_AFTER_RULE_DEFS);
-
- assertThat(target.computeRuleVariables().get(NinjaRuleVariable.DESCRIPTION))
- .isEqualTo("bunny bunny");
- }
-
- @Test
- public void testTargetMayNotOverrideCommand() throws Exception {
- NinjaScope scope = new NinjaScope();
-
- NinjaParserStep parser = createParser("rule testRule \n command = foo\n");
- NinjaRule ninjaRule = parser.parseNinjaRule();
- scope.setRules(ImmutableSortedMap.of("testRule", ImmutableList.of(Pair.of(0L, ninjaRule))));
-
- NinjaTarget target =
- createParser("build output : testRule input\n command = bar\n")
- .parseNinjaTarget(scope, LINE_NUM_AFTER_RULE_DEFS);
-
- // Verify the target's COMMAND rule variable is the one defined by the rule, not the one
- // defined by the target.
- assertThat(target.computeRuleVariables().get(NinjaRuleVariable.COMMAND)).isEqualTo("foo");
- }
-
- private static NinjaScope scopeWithStubRule(String ruleName) throws Exception {
- NinjaScope scope = new NinjaScope();
-
- NinjaParserStep parser = createParser("rule " + ruleName + " \n command = foo\n");
- NinjaRule ninjaRule = parser.parseNinjaRule();
- scope.setRules(ImmutableSortedMap.of(ruleName, ImmutableList.of(Pair.of(0L, ninjaRule))));
- return scope;
- }
-
- private static void testNinjaTargetParsingError(String text, String error) {
- GenericParsingException exception =
- assertThrows(GenericParsingException.class, () -> parseNinjaTarget(text));
- assertThat(exception).hasMessageThat().isEqualTo(error);
- }
-
- private static NinjaTarget parseNinjaTarget(String text) throws Exception {
- NinjaScope fileScope = new NinjaScope();
- return createParser(text).parseNinjaTarget(fileScope, 0);
- }
-
- private static void doTestNinjaRuleParsingException(String text, String message) {
- GenericParsingException exception =
- assertThrows(GenericParsingException.class, () -> createParser(text).parseNinjaRule());
- assertThat(exception).hasMessageThat().isEqualTo(message);
- }
-
- private static void doTestSimpleVariable(String text, String name, String value)
- throws Exception {
- NinjaParserStep parser = createParser(text);
- Pair<String, NinjaVariableValue> variable = parser.parseVariable();
- assertThat(variable.getFirst()).isEqualTo(name);
- assertThat(variable.getSecond()).isNotNull();
- assertThat(variable.getSecond().getRawText()).isEqualTo(value);
-
- MockValueExpander expander = new MockValueExpander("###");
- assertThat(variable.getSecond().getExpandedValue(expander)).isEqualTo(value);
- assertThat(expander.getRequestedVariables()).isEmpty();
- }
-
- private static void doTestNoValue(String text) throws Exception {
- NinjaParserStep parser = createParser(text);
- NinjaVariableValue value = parser.parseVariable().getSecond();
- assertThat(value).isNotNull();
- assertThat(value.getRawText()).isEmpty();
- }
-
- private static void doTestWithVariablesInValue(
- String text, String name, String value, ImmutableSortedSet<String> expectedVars)
- throws Exception {
- NinjaParserStep parser = createParser(text);
- Pair<String, NinjaVariableValue> variable = parser.parseVariable();
- assertThat(variable.getFirst()).isEqualTo(name);
- assertThat(variable.getSecond()).isNotNull();
- assertThat(variable.getSecond().getRawText()).isEqualTo(value);
-
- MockValueExpander expander = new MockValueExpander("###");
- assertThat(variable.getSecond().getExpandedValue(expander)).contains("###");
- assertThat(expander.getRequestedVariables()).containsExactlyElementsIn(expectedVars);
- }
-
- private static NinjaParserStep createParser(String text) {
- ByteBuffer buffer = ByteBuffer.wrap(text.getBytes(StandardCharsets.ISO_8859_1));
- NinjaLexer lexer = new NinjaLexer(new FileFragment(buffer, 0, 0, buffer.limit()));
- return new NinjaParserStep(
- lexer, BlazeInterners.newWeakInterner(), BlazeInterners.newWeakInterner());
- }
-
- private static class MockValueExpander implements Function<String, String> {
- private final ImmutableSortedSet.Builder<String> setBuilder;
- private final String prefix;
-
- private MockValueExpander(String prefix) {
- this.prefix = prefix;
- setBuilder = ImmutableSortedSet.naturalOrder();
- }
-
- @Override
- public String apply(String s) {
- setBuilder.add(s);
- return prefix + s;
- }
-
- public ImmutableSortedSet<String> getRequestedVariables() {
- return setBuilder.build();
- }
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaPhonyTargetsUtilTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaPhonyTargetsUtilTest.java
deleted file mode 100644
index 7473a9e..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaPhonyTargetsUtilTest.java
+++ /dev/null
@@ -1,172 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja;
-
-import static com.google.common.truth.Truth.assertThat;
-import static org.junit.Assert.assertThrows;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.Iterables;
-import com.google.devtools.build.lib.bazel.rules.ninja.actions.NinjaPhonyTargetsUtil;
-import com.google.devtools.build.lib.bazel.rules.ninja.actions.PhonyTarget;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaParserStep;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaScope;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget;
-import com.google.devtools.build.lib.concurrent.BlazeInterners;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.Set;
-import java.util.stream.Collectors;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * Tests for {@link com.google.devtools.build.lib.bazel.rules.ninja.actions.NinjaPhonyTargetsUtil}.
- */
-@RunWith(JUnit4.class)
-public class NinjaPhonyTargetsUtilTest {
- @Test
- public void testPathsTree() throws Exception {
- ImmutableList<String> targetTexts =
- ImmutableList.of(
- "build alias9: phony alias2 alias3 direct1 direct2",
- "build alias2: phony direct3 direct4",
- "build alias3: phony alias4 direct4 direct5",
- "build alias4: phony alias2");
-
- ImmutableSortedMap<PathFragment, PhonyTarget> pathsMap =
- NinjaPhonyTargetsUtil.getPhonyPathsMap(buildPhonyTargets(targetTexts));
-
- assertThat(pathsMap).hasSize(4);
- checkMapping(pathsMap, "alias9", "direct1", "direct2", "direct3", "direct4", "direct5");
- checkMapping(pathsMap, "alias2", "direct3", "direct4");
- checkMapping(pathsMap, "alias3", "direct3", "direct4", "direct5");
- checkMapping(pathsMap, "alias4", "direct3", "direct4");
- }
-
- @Test
- public void testDag() throws Exception {
- ImmutableList<String> targetTexts =
- ImmutableList.of(
- "build _alias9: phony alias1 alias2",
- "build alias1: phony deep1",
- "build alias2: phony deep2",
- "build deep1: phony leaf1",
- "build deep2: phony leaf2 alias1");
-
- ImmutableSortedMap<PathFragment, PhonyTarget> pathsMap =
- NinjaPhonyTargetsUtil.getPhonyPathsMap(buildPhonyTargets(targetTexts));
-
- assertThat(pathsMap).hasSize(5);
- checkMapping(pathsMap, "_alias9", "leaf1", "leaf2");
- checkMapping(pathsMap, "alias1", "leaf1");
- checkMapping(pathsMap, "alias2", "leaf1", "leaf2");
- checkMapping(pathsMap, "deep1", "leaf1");
- checkMapping(pathsMap, "deep2", "leaf1", "leaf2");
- }
-
- @Test
- public void testAlwaysDirty() throws Exception {
- ImmutableList<String> targetTexts =
- ImmutableList.of(
- "build alias9: phony alias2 alias3 direct1 direct2",
- "build alias2: phony direct3 direct4",
- "build alias3: phony alias4 direct4 direct5",
- // alias4 is always-dirty
- "build alias4: phony");
-
- ImmutableSortedMap<PathFragment, PhonyTarget> pathsMap =
- NinjaPhonyTargetsUtil.getPhonyPathsMap(buildPhonyTargets(targetTexts));
-
- assertThat(pathsMap).hasSize(4);
- // alias4 and its transitive closure is always-dirty
- checkMapping(pathsMap, "alias9", true, "direct1", "direct2", "direct3", "direct4", "direct5");
- checkMapping(pathsMap, "alias2", false, "direct3", "direct4");
- checkMapping(pathsMap, "alias3", true, "direct4", "direct5");
- checkMapping(pathsMap, "alias4", true);
- }
-
- private static void checkMapping(
- ImmutableSortedMap<PathFragment, PhonyTarget> pathsMap, String key, String... values) {
- checkMapping(pathsMap, key, false, values);
- }
-
- private static void checkMapping(
- ImmutableSortedMap<PathFragment, PhonyTarget> pathsMap,
- String key,
- boolean isAlwaysDirty,
- String... values) {
- Set<PathFragment> expectedPaths =
- Arrays.stream(values).map(PathFragment::create).collect(Collectors.toSet());
- PhonyTarget phonyTarget = pathsMap.get(PathFragment.create(key));
- assertThat(phonyTarget).isNotNull();
- assertThat(phonyTarget.isAlwaysDirty()).isEqualTo(isAlwaysDirty);
-
- ImmutableSortedSet.Builder<PathFragment> paths = ImmutableSortedSet.naturalOrder();
- pathsMap.get(PathFragment.create(key)).visitExplicitInputs(pathsMap, paths::addAll);
- assertThat(paths.build()).containsExactlyElementsIn(expectedPaths);
- }
-
- private static ImmutableSortedMap<PathFragment, NinjaTarget> buildPhonyTargets(
- ImmutableList<String> targetTexts) throws Exception {
- ImmutableSortedMap.Builder<PathFragment, NinjaTarget> builder =
- ImmutableSortedMap.naturalOrder();
- for (String text : targetTexts) {
- NinjaTarget ninjaTarget = parseNinjaTarget(text);
- builder.put(Iterables.getOnlyElement(ninjaTarget.getAllOutputs()), ninjaTarget);
- }
- return builder.buildOrThrow();
- }
-
- @Test
- public void testEmptyMap() throws Exception {
- assertThat(NinjaPhonyTargetsUtil.getPhonyPathsMap(ImmutableSortedMap.of())).isEmpty();
- }
-
- @Test
- public void testCycle() {
- ImmutableList<String> targetTexts =
- ImmutableList.of(
- "build alias1: phony alias2 direct1", "build alias2: phony alias1 direct2");
-
- GenericParsingException exception =
- assertThrows(
- GenericParsingException.class,
- () -> NinjaPhonyTargetsUtil.getPhonyPathsMap(buildPhonyTargets(targetTexts)));
- assertThat(exception)
- .hasMessageThat()
- .isEqualTo("Detected a dependency cycle involving the phony target 'alias1'");
- }
-
- private static NinjaTarget parseNinjaTarget(String text) throws Exception {
- NinjaScope fileScope = new NinjaScope();
- return createParser(text).parseNinjaTarget(fileScope, 0);
- }
-
- private static NinjaParserStep createParser(String text) {
- ByteBuffer buffer = ByteBuffer.wrap(text.getBytes(StandardCharsets.ISO_8859_1));
- NinjaLexer lexer = new NinjaLexer(new FileFragment(buffer, 0, 0, buffer.limit()));
- return new NinjaParserStep(
- lexer, BlazeInterners.newWeakInterner(), BlazeInterners.newWeakInterner());
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaPipelineTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaPipelineTest.java
deleted file mode 100644
index 40ac34e..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaPipelineTest.java
+++ /dev/null
@@ -1,319 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.rules.ninja;
-
-import static com.google.common.truth.Truth.assertThat;
-import static junit.framework.TestCase.fail;
-import static org.junit.Assert.assertThrows;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaRuleVariable;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaTarget;
-import com.google.devtools.build.lib.bazel.rules.ninja.pipeline.NinjaPipelineImpl;
-import com.google.devtools.build.lib.concurrent.ExecutorUtil;
-import com.google.devtools.build.lib.vfs.DigestHashFunction;
-import com.google.devtools.build.lib.vfs.FileSystemUtils;
-import com.google.devtools.build.lib.vfs.JavaIoFileSystem;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.List;
-import java.util.concurrent.Executors;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link com.google.devtools.build.lib.bazel.rules.ninja.pipeline.NinjaPipelineImpl}. */
-@RunWith(JUnit4.class)
-public class NinjaPipelineTest {
- private static class Tester {
- private final Path dir;
- private final ListeningExecutorService service;
-
- Tester() throws IOException {
- service =
- MoreExecutors.listeningDecorator(
- Executors.newFixedThreadPool(
- 25,
- new ThreadFactoryBuilder()
- .setNameFormat(NinjaPipelineTest.class.getSimpleName() + "-%d")
- .build()));
- java.nio.file.Path tmpDir = Files.createTempDirectory("test");
- dir = new JavaIoFileSystem(DigestHashFunction.SHA256).getPath(tmpDir.toString());
- }
-
- ListeningExecutorService getService() {
- return service;
- }
-
- Path writeTmpFile(String name, String... lines) throws IOException {
- Path path = dir.getRelative(name);
- if (lines.length > 0) {
- FileSystemUtils.writeContent(
- path, String.join("\n", lines).getBytes(StandardCharsets.ISO_8859_1));
- } else {
- FileSystemUtils.createEmptyFile(path);
- }
- return path;
- }
-
- public void tearDown() throws IOException {
- ExecutorUtil.interruptibleShutdown(service);
- dir.deleteTree();
- }
- }
-
- private Tester tester;
-
- @Before
- public void setUp() throws Exception {
- tester = new Tester();
- }
-
- @After
- public void tearDown() throws Exception {
- tester.tearDown();
- }
-
- @Test
- public void testOneFilePipeline() throws Exception {
- Path vfsPath =
- tester.writeTmpFile(
- "test.ninja",
- "rule r1",
- " command = c $in $out",
- "build t1: r1 in1 in2",
- "build t2: r1 in3");
- NinjaPipelineImpl pipeline =
- new NinjaPipelineImpl(
- vfsPath.getParentDirectory(), tester.getService(), ImmutableList.of(), "ninja_target");
- List<NinjaTarget> targets = pipeline.pipeline(vfsPath);
- checkTargets(targets);
- }
-
- @Test
- public void testOneFilePipelineWithNewlines() throws Exception {
- Path vfsPath =
- tester.writeTmpFile(
- "test.ninja",
- "",
- "",
- "",
- "",
- "rule r1",
- " command = c $in $out",
- "",
- "",
- "",
- "build t1: r1 in1 in2",
- "",
- "",
- "",
- "build t2: r1 in3",
- "");
- NinjaPipelineImpl pipeline =
- new NinjaPipelineImpl(
- vfsPath.getParentDirectory(), tester.getService(), ImmutableList.of(), "ninja_target");
- List<NinjaTarget> targets = pipeline.pipeline(vfsPath);
- checkTargets(targets);
- }
-
- @Test
- public void testWithIncluded() throws Exception {
- Path vfsPath =
- tester.writeTmpFile(
- "test.ninja", "rule r1", " command = c $in $out", "include child.ninja");
- Path childFile = tester.writeTmpFile("child.ninja", "build t1: r1 in1 in2", "build t2: r1 in3");
- NinjaPipelineImpl pipeline =
- new NinjaPipelineImpl(
- vfsPath.getParentDirectory(),
- tester.getService(),
- ImmutableList.of(childFile),
- "ninja_target");
- List<NinjaTarget> targets = pipeline.pipeline(vfsPath);
- checkTargets(targets);
- }
-
- @Test
- public void testComputedSubNinja() throws Exception {
- Path vfsPath =
- tester.writeTmpFile(
- "test.ninja",
- "subfile=child",
- "rule r1",
- " command = c $in $out",
- "include ${subfile}.ninja");
- Path childFile = tester.writeTmpFile("child.ninja", "build t1: r1 in1 in2", "build t2: r1 in3");
- NinjaPipelineImpl pipeline =
- new NinjaPipelineImpl(
- vfsPath.getParentDirectory(),
- tester.getService(),
- ImmutableList.of(childFile),
- "ninja_target");
- List<NinjaTarget> targets = pipeline.pipeline(vfsPath);
- checkTargets(targets);
- }
-
- @Test
- public void testComputedDeeplyIncluded() throws Exception {
- Path vfsPath =
- tester.writeTmpFile(
- "test.ninja",
- "subfile=child",
- "subninja_file=sub",
- "rule r1",
- " command = c $in $out",
- "include ${subfile}.ninja",
- "build t1: r1 ${top_scope_var} in2");
- Path childFile =
- tester.writeTmpFile(
- "child.ninja",
- "top_scope_var=in1",
- "var_for_sub=in3",
- "subninja ${subninja_file}.ninja");
- Path subFile = tester.writeTmpFile("sub.ninja", "build t2: r1 ${var_for_sub}");
- NinjaPipelineImpl pipeline =
- new NinjaPipelineImpl(
- vfsPath.getParentDirectory(),
- tester.getService(),
- ImmutableList.of(childFile, subFile),
- "ninja_target");
- List<NinjaTarget> targets = pipeline.pipeline(vfsPath);
- checkTargets(targets);
- }
-
- @Test
- public void testEmptyFile() throws Exception {
- Path vfsPath = tester.writeTmpFile("test.ninja");
- NinjaPipelineImpl pipeline =
- new NinjaPipelineImpl(
- vfsPath.getParentDirectory(), tester.getService(), ImmutableList.of(), "ninja_target");
- List<NinjaTarget> targets = pipeline.pipeline(vfsPath);
- assertThat(targets).isEmpty();
- }
-
- @Test
- public void testIncludedNinjaFileIsNotDeclared() throws Exception {
- Path vfsPath = tester.writeTmpFile("test.ninja", "include subfile.ninja");
- GenericParsingException exception =
- assertThrows(
- GenericParsingException.class,
- () ->
- new NinjaPipelineImpl(
- vfsPath.getParentDirectory(),
- tester.getService(),
- ImmutableList.of(),
- "ninja_target")
- .pipeline(vfsPath));
- assertThat(exception)
- .hasMessageThat()
- .isEqualTo(
- "Ninja file 'subfile.ninja' requested from 'test.ninja' "
- + "not declared in 'ninja_srcs' attribute of 'ninja_target'.");
- }
-
- @Test
- public void testIncludeCycle() throws Exception {
- Path vfsPath = tester.writeTmpFile("test.ninja", "include one.ninja");
- Path oneFile = tester.writeTmpFile("one.ninja", "include two.ninja");
- Path twoFile = tester.writeTmpFile("two.ninja", "include one.ninja");
- NinjaPipelineImpl pipeline =
- new NinjaPipelineImpl(
- vfsPath.getParentDirectory(),
- tester.getService(),
- ImmutableList.of(oneFile, twoFile),
- "ninja_target");
- GenericParsingException exception =
- assertThrows(GenericParsingException.class, () -> pipeline.pipeline(vfsPath));
- assertThat(exception)
- .hasMessageThat()
- .isEqualTo(
- "Detected cycle or duplicate inclusion in Ninja files dependencies, "
- + "including 'one.ninja'.");
- }
-
- @Test
- public void testSpacesInEmptyLine() throws Exception {
- Path vfsPath =
- tester.writeTmpFile(
- "test.ninja",
- "subfile=child",
- " ",
- "subninja_file=sub",
- " ",
- "rule r1",
- " command = c $in $out",
- // This could be interpreted as indent, but should be skipped as the empty line.
- " ",
- " \n\n\n\n",
- "include ${subfile}.ninja",
- " ",
- "build t2: r1 in3",
- " ",
- "build t1: r1 in1 in2");
- Path childFile = tester.writeTmpFile("child.ninja");
- NinjaPipelineImpl pipeline =
- new NinjaPipelineImpl(
- vfsPath.getParentDirectory(),
- tester.getService(),
- ImmutableList.of(childFile),
- "ninja_target");
- List<NinjaTarget> targets = pipeline.pipeline(vfsPath);
- checkTargets(targets);
- }
-
- @Test
- public void testBigFile() throws Exception {
- String[] lines = new String[1000];
- for (int i = 0; i < lines.length - 2; i++) {
- lines[i] = "rule rule" + i + "\n command = echo 'Hello' > ${out}";
- }
- lines[998] = "build out: rule1";
- lines[999] = "pool link_pool\n depth = 4";
- Path path = tester.writeTmpFile("big_file.ninja", lines);
- NinjaPipelineImpl pipeline =
- new NinjaPipelineImpl(
- path.getParentDirectory(), tester.getService(), ImmutableList.of(), "ninja_target");
- // Test specifically that all manipulations with connecting buffers are working fine:
- // for that, have relatively long file and small buffer size.
- pipeline.setReadBlockSize(100);
- List<NinjaTarget> targets = pipeline.pipeline(path);
- assertThat(targets).hasSize(1);
- assertThat(targets.get(0).computeRuleVariables().get(NinjaRuleVariable.COMMAND))
- .isEqualTo("echo 'Hello' > out");
- }
-
- private static void checkTargets(List<NinjaTarget> targets) {
- assertThat(targets).hasSize(2);
- for (NinjaTarget target : targets) {
- if (target.getAllOutputs().contains(PathFragment.create("t1"))) {
- assertThat(target.getAllInputs())
- .containsExactly(PathFragment.create("in1"), PathFragment.create("in2"));
- } else if (target.getAllOutputs().contains(PathFragment.create("t2"))) {
- assertThat(target.getAllInputs()).containsExactly(PathFragment.create("in3"));
- } else {
- fail();
- }
- }
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaScopeTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaScopeTest.java
deleted file mode 100644
index bba3f64..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaScopeTest.java
+++ /dev/null
@@ -1,315 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja;
-
-import static com.google.common.truth.Truth.assertThat;
-import static org.junit.Assert.assertThrows;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Maps;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.lexer.NinjaLexer;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaFileParseResult;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaParserStep;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaRule;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaRuleVariable;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaScope;
-import com.google.devtools.build.lib.bazel.rules.ninja.parser.NinjaVariableValue;
-import com.google.devtools.build.lib.concurrent.BlazeInterners;
-import com.google.devtools.build.lib.util.Pair;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.stream.Collectors;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link NinjaScope} */
-@RunWith(JUnit4.class)
-public class NinjaScopeTest {
- @Test
- public void testSortVariables() {
- NinjaFileParseResult parseResult = new NinjaFileParseResult();
- parseResult.addVariable("abc", 12, NinjaVariableValue.createPlainText("cba"));
- parseResult.addVariable("abc", 1, NinjaVariableValue.createPlainText("cba1"));
- parseResult.addVariable("abc", 14, NinjaVariableValue.createPlainText("cba2"));
-
- parseResult.sortResults();
-
- List<Long> offsets =
- parseResult.getVariables().get("abc").stream()
- .map(Pair::getFirst)
- .collect(Collectors.toList());
- assertThat(offsets).isInOrder();
- }
-
- @Test
- public void testSortRules() {
- // We can just use the same rule value here.
- NinjaRule rule = rule("rule1");
-
- NinjaFileParseResult parseResult = new NinjaFileParseResult();
- parseResult.addRule(10, rule);
- parseResult.addRule(1115, rule);
- parseResult.addRule(5, rule);
-
- parseResult.sortResults();
-
- List<Long> offsets =
- parseResult.getRules().get(rule.getName()).stream()
- .map(Pair::getFirst)
- .collect(Collectors.toList());
- assertThat(offsets).isInOrder();
- }
-
- @Test
- public void testMerge() {
- NinjaRule rule1 = rule("rule1");
- NinjaRule rule2 = rule("rule2");
-
- NinjaFileParseResult parseResult1 = new NinjaFileParseResult();
- parseResult1.addRule(10, rule1);
- parseResult1.addVariable("from1", 7, NinjaVariableValue.createPlainText("111"));
- parseResult1.addVariable("abc", 5, NinjaVariableValue.createPlainText("5"));
- parseResult1.addVariable("abc", 115, NinjaVariableValue.createPlainText("7"));
-
- NinjaFileParseResult parseResult2 = new NinjaFileParseResult();
- parseResult2.addRule(10, rule2);
- parseResult2.addVariable("from2", 20017, NinjaVariableValue.createPlainText("222"));
- parseResult2.addVariable("abc", 2005, NinjaVariableValue.createPlainText("15"));
- parseResult2.addVariable("abc", 20015, NinjaVariableValue.createPlainText("17"));
-
- NinjaFileParseResult result =
- NinjaFileParseResult.merge(ImmutableList.of(parseResult1, parseResult2));
- assertThat(result.getRules()).hasSize(2);
- assertThat(result.getRules()).containsKey("rule1");
- assertThat(result.getRules()).containsKey("rule2");
-
- assertThat(result.getVariables()).hasSize(3);
- assertThat(result.getVariables()).containsKey("from1");
- assertThat(result.getVariables()).containsKey("from2");
- assertThat(result.getVariables()).containsKey("abc");
-
- List<Pair<Long, NinjaVariableValue>> abc = result.getVariables().get("abc");
- assertThat(abc).hasSize(4);
- assertThat(abc.stream().map(Pair::getFirst).collect(Collectors.toList())).isInOrder();
- }
-
- @Test
- public void testFindVariable() throws Exception {
- NinjaFileParseResult parseResult = new NinjaFileParseResult();
- parseResult.addVariable("abc", 12, NinjaVariableValue.createPlainText("cba"));
- parseResult.addVariable("abc", 5, NinjaVariableValue.createPlainText("cba1"));
- parseResult.addVariable("abc", 14, NinjaVariableValue.createPlainText("cba2"));
-
- parseResult.sortResults();
- NinjaScope scope = new NinjaScope();
- parseResult.expandIntoScope(scope, Maps.newHashMap());
-
- assertThat(scope.findExpandedVariable(1, "not_there")).isNull();
- assertThat(scope.findExpandedVariable(1, "abc")).isNull();
- assertThat(scope.findExpandedVariable(6, "abc")).isEqualTo("cba1");
-
- assertThat(scope.findExpandedVariable(13, "abc")).isEqualTo("cba");
- assertThat(scope.findExpandedVariable(130, "abc")).isEqualTo("cba2");
- }
-
- @Test
- public void testFindVariableErrors() throws Exception {
- NinjaFileParseResult parseResult = new NinjaFileParseResult();
- parseResult.addVariable("abc", 12, NinjaVariableValue.createPlainText("cba"));
- parseResult.addVariable("abc", 5, NinjaVariableValue.createPlainText("cba1"));
- parseResult.addVariable("abc", 14, NinjaVariableValue.createPlainText("cba2"));
-
- parseResult.sortResults();
- NinjaScope scope = new NinjaScope();
- parseResult.expandIntoScope(scope, Maps.newHashMap());
-
- IllegalStateException exception =
- assertThrows(IllegalStateException.class, () -> scope.findExpandedVariable(5, "abc"));
- assertThat(exception)
- .hasMessageThat()
- .isEqualTo("Trying to interpret declaration as reference.");
- }
-
- @Test
- public void testFindRule() throws Exception {
- NinjaFileParseResult parseResult = new NinjaFileParseResult();
- parseResult.addRule(10, rule("rule1", "10"));
- parseResult.addRule(1115, rule("rule1", "1115"));
- parseResult.addRule(5, rule("rule1", "5"));
-
- parseResult.sortResults();
- NinjaScope scope = new NinjaScope();
- parseResult.expandIntoScope(scope, Maps.newHashMap());
-
- assertThat(scope.findRule(1, "non-existent")).isNull();
- assertThat(scope.findRule(1, "rule1")).isNull();
-
- NinjaRule rule1 = scope.findRule(6, "rule1");
- assertThat(rule1).isNotNull();
- assertThat(rule1.getVariables().get(NinjaRuleVariable.COMMAND).getRawText()).isEqualTo("5");
-
- rule1 = scope.findRule(15, "rule1");
- assertThat(rule1).isNotNull();
- assertThat(rule1.getVariables().get(NinjaRuleVariable.COMMAND).getRawText()).isEqualTo("10");
- }
-
- @Test
- public void testFindVariableInParentScope() throws Exception {
- NinjaFileParseResult parentParseResult = new NinjaFileParseResult();
- parentParseResult.addVariable("abc", 12, NinjaVariableValue.createPlainText("abc"));
- parentParseResult.addVariable("edf", 120, NinjaVariableValue.createPlainText("edf"));
- parentParseResult.addVariable("xyz", 1000, NinjaVariableValue.createPlainText("xyz"));
-
- // This is subninja scope, not include scope.
- NinjaFileParseResult childParseResult = new NinjaFileParseResult();
- parentParseResult.addSubNinjaScope(140, scope -> childParseResult);
- // Shadows this variable from parent.
- childParseResult.addVariable("edf", 1, NinjaVariableValue.createPlainText("11111"));
-
- parentParseResult.sortResults();
- NinjaScope scope = new NinjaScope();
- parentParseResult.expandIntoScope(scope, Maps.newHashMap());
-
- assertThat(scope.getSubNinjaScopes()).hasSize(1);
- NinjaScope child = scope.getSubNinjaScopes().iterator().next();
-
- assertThat(child.findExpandedVariable(2, "abc")).isEqualTo("abc");
- assertThat(child.findExpandedVariable(2, "edf")).isEqualTo("11111");
- assertThat(child.findExpandedVariable(2, "xyz")).isNull();
- }
-
- @Test
- public void testfindExpandedVariableInIncludedScope() throws Exception {
- NinjaFileParseResult parentParseResult = new NinjaFileParseResult();
- parentParseResult.addVariable("abc", 12, NinjaVariableValue.createPlainText("abc"));
- parentParseResult.addVariable("edf", 120, NinjaVariableValue.createPlainText("edf"));
- parentParseResult.addVariable("xyz", 1000, NinjaVariableValue.createPlainText("xyz"));
-
- NinjaFileParseResult childParseResult = new NinjaFileParseResult();
- parentParseResult.addIncludeScope(140, scope -> childParseResult);
- // Shadows this variable from parent.
- childParseResult.addVariable("edf", 1, NinjaVariableValue.createPlainText("11111"));
- childParseResult.addVariable("child", 2, NinjaVariableValue.createPlainText("child"));
-
- NinjaFileParseResult childParseResult2 = new NinjaFileParseResult();
- parentParseResult.addIncludeScope(200, scope -> childParseResult2);
- childParseResult2.addVariable("edf", 1, NinjaVariableValue.createPlainText("22222"));
-
- parentParseResult.sortResults();
- NinjaScope scope = new NinjaScope();
- parentParseResult.expandIntoScope(scope, Maps.newHashMap());
-
- assertThat(scope.findExpandedVariable(160, "edf")).isEqualTo("11111");
- assertThat(scope.findExpandedVariable(220, "edf")).isEqualTo("22222");
- assertThat(scope.findExpandedVariable(125, "edf")).isEqualTo("edf");
- assertThat(scope.findExpandedVariable(145, "child")).isEqualTo("child");
- }
-
- @Test
- public void testFindInRecursivelyIncluded() throws Exception {
- NinjaFileParseResult parentParseResult = new NinjaFileParseResult();
- parentParseResult.addVariable("abc", 12, NinjaVariableValue.createPlainText("abc"));
- parentParseResult.addVariable("edf", 120, NinjaVariableValue.createPlainText("edf"));
- parentParseResult.addVariable("xyz", 1000, NinjaVariableValue.createPlainText("xyz"));
-
- NinjaFileParseResult childParseResult1 = new NinjaFileParseResult();
- parentParseResult.addIncludeScope(140, scope -> childParseResult1);
- // Shadows this variable from parent.
- childParseResult1.addVariable("edf", 1, NinjaVariableValue.createPlainText("11111"));
- childParseResult1.addVariable("child", 2, NinjaVariableValue.createPlainText("child"));
-
- NinjaFileParseResult childParseResult2 = new NinjaFileParseResult();
- childParseResult1.addIncludeScope(3, scope -> childParseResult2);
- childParseResult2.addVariable("edf", 1, NinjaVariableValue.createPlainText("22222"));
-
- parentParseResult.sortResults();
- NinjaScope scope = new NinjaScope();
- parentParseResult.expandIntoScope(scope, Maps.newHashMap());
-
- assertThat(scope.findExpandedVariable(220, "edf")).isEqualTo("22222");
- }
-
- @Test
- public void testVariableExpand() throws Exception {
- NinjaFileParseResult parseResult = new NinjaFileParseResult();
- parseResult.addVariable("abc", 12, NinjaVariableValue.createPlainText("abc"));
- parseResult.addVariable("edf", 120, parseValue("=> $abc = ?"));
- parseResult.addVariable("abc", 130, NinjaVariableValue.createPlainText("redefined"));
- parseResult.addVariable("edf", 180, parseValue("now$: $abc!"));
-
- parseResult.sortResults();
- NinjaScope scope = new NinjaScope();
- parseResult.expandIntoScope(scope, Maps.newHashMap());
-
- assertThat(scope.findExpandedVariable(15, "abc")).isEqualTo("abc");
- assertThat(scope.findExpandedVariable(150, "edf")).isEqualTo("=> abc = ?");
- assertThat(scope.findExpandedVariable(140, "abc")).isEqualTo("redefined");
- assertThat(scope.findExpandedVariable(181, "edf")).isEqualTo("now: redefined!");
- }
-
- @Test
- public void testExpandWithParentChild() throws Exception {
- NinjaFileParseResult parentParseResult = new NinjaFileParseResult();
- parentParseResult.addVariable("abc", 12, NinjaVariableValue.createPlainText("abc"));
- parentParseResult.addVariable("edf", 120, parseValue("$abc === ${ abc }"));
-
- NinjaFileParseResult includeParseResult = new NinjaFileParseResult();
- parentParseResult.addIncludeScope(140, scope -> includeParseResult);
- includeParseResult.addVariable("included", 1, parseValue("<$abc and ${ edf }>"));
-
- NinjaFileParseResult childParseResult = new NinjaFileParseResult();
- parentParseResult.addSubNinjaScope(150, scope -> childParseResult);
- childParseResult.addVariable("subninja", 2, parseValue("$edf = ${ included }*"));
-
- parentParseResult.sortResults();
- NinjaScope parentScope = new NinjaScope();
- parentParseResult.expandIntoScope(parentScope, Maps.newHashMap());
-
- assertThat(parentScope.getIncludedScopes()).hasSize(1);
- NinjaScope includeScope = parentScope.getIncludedScopes().iterator().next();
- assertThat(parentScope.getSubNinjaScopes()).hasSize(1);
- NinjaScope childScope = parentScope.getSubNinjaScopes().iterator().next();
-
- assertThat(includeScope.findExpandedVariable(2, "included")).isEqualTo("<abc and abc === abc>");
- assertThat(childScope.findExpandedVariable(3, "subninja"))
- .isEqualTo("abc === abc = <abc and abc === abc>*");
- assertThat(parentScope.findExpandedVariable(150, "included"))
- .isEqualTo("<abc and abc === abc>");
- }
-
- private static NinjaRule rule(String name) {
- return rule(name, "command");
- }
-
- private static NinjaRule rule(String name, String command) {
- return new NinjaRule(
- name,
- ImmutableSortedMap.of(
- NinjaRuleVariable.COMMAND, NinjaVariableValue.createPlainText(command)));
- }
-
- private static NinjaVariableValue parseValue(String text) throws Exception {
- ByteBuffer bb = ByteBuffer.wrap(text.getBytes(StandardCharsets.ISO_8859_1));
- NinjaLexer lexer = new NinjaLexer(new FileFragment(bb, 0, 0, bb.limit()));
- return new NinjaParserStep(
- lexer, BlazeInterners.newWeakInterner(), BlazeInterners.newWeakInterner())
- .parseVariableValue();
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaSeparatorFinderTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaSeparatorFinderTest.java
deleted file mode 100644
index 207408e..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/NinjaSeparatorFinderTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja;
-
-import static com.google.common.truth.Truth.assertThat;
-import static org.junit.Assert.assertThrows;
-
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.IncorrectSeparatorException;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.NinjaSeparatorFinder;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@link NinjaSeparatorFinder}. */
-@RunWith(JUnit4.class)
-public class NinjaSeparatorFinderTest {
- @Test
- public void testIsSeparator() throws IncorrectSeparatorException {
- doTestIsSeparator(" \na", 1);
- doTestIsSeparator("b\na", 1);
- doTestIsSeparator(" \na", 1);
- doTestIsSeparator("b\n$", 1);
- doTestIsSeparator(" \n\n", 1);
- doTestIsSeparator("a\n\n", 1);
- // We are pointing to the last symbol of separator.
- doTestIsSeparator("a\r\n\n", 2);
- doTestIsSeparator(" \r\n\n", 2);
- doTestIsSeparator("a\r\na", 2);
- doTestIsSeparator("\r\na", 1);
-
- doTestIsSeparator(" \n ", -1);
- doTestIsSeparator(" \r\n ", -1);
- doTestIsSeparator("$\n ", -1);
- doTestIsSeparator("$\r\n ", -1);
- doTestIsSeparator("$\r\n\n ", -1);
- doTestIsSeparator("$\r\n\r\n ", -1);
- doTestIsSeparator("$\n\n", -1);
- doTestIsSeparator("$\na", -1);
- doTestIsSeparator("$\r\na", -1);
- doTestIsSeparator("a\n ", -1);
- doTestIsSeparator("a\n\t", -1);
- // Not enough information.
- doTestIsSeparator("\r\n", -1);
- doTestIsSeparator("\n", -1);
- // Test for incorrect separators.
- byte[] bytes = "a\rb".getBytes(StandardCharsets.ISO_8859_1);
- ByteBuffer buffer = ByteBuffer.wrap(bytes);
- FileFragment fragment = new FileFragment(buffer, 0, 0, buffer.limit());
- assertThrows(
- IncorrectSeparatorException.class,
- () -> NinjaSeparatorFinder.findNextSeparator(fragment, 0, -1));
- }
-
- private static void doTestIsSeparator(String s, int expected) throws IncorrectSeparatorException {
- byte[] bytes = s.getBytes(StandardCharsets.ISO_8859_1);
- ByteBuffer buffer = ByteBuffer.wrap(bytes);
- FileFragment fragment = new FileFragment(buffer, 0, 0, buffer.limit());
- int result = NinjaSeparatorFinder.findNextSeparator(fragment, 0, -1);
- assertThat(result).isEqualTo(expected);
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/ParallelFileProcessingTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/ParallelFileProcessingTest.java
deleted file mode 100644
index 7a3c755..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/rules/ninja/ParallelFileProcessingTest.java
+++ /dev/null
@@ -1,249 +0,0 @@
-// Copyright 2019 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package com.google.devtools.build.lib.bazel.rules.ninja;
-
-import static com.google.common.truth.Truth.assertThat;
-
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import com.google.common.io.Files;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.DeclarationConsumer;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.FileFragment;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.GenericParsingException;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ParallelFileProcessing;
-import com.google.devtools.build.lib.bazel.rules.ninja.file.ParallelFileProcessing.BlockParameters;
-import com.google.devtools.build.lib.concurrent.ExecutorUtil;
-import java.io.File;
-import java.io.IOException;
-import java.nio.channels.SeekableByteChannel;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.Executors;
-import java.util.function.Supplier;
-import java.util.stream.IntStream;
-import javax.annotation.Nullable;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * Tests for {@link com.google.devtools.build.lib.bazel.rules.ninja.file.ParallelFileProcessing}.
- *
- * <p>{@link #doPerformanceTest(File)} sample results for the "parallel buffers processing" solution
- * motivation, showing significant improvement over straightforward implementation:
- *
- * <p>File size: 1,321,144 KB Parallel run: 727 ms (704 ms, 738 ms, 739 ms) Files.readLines() run:
- * 7344 ms (7888 ms, 7244 ms, 6900 ms)
- *
- * <p>File size: 959,521 KB Parallel run: 524 ms (539 ms, 546 ms, 486 ms) Files.readLines() run:
- * 5520 ms (5161 ms, 5350 ms, 6048 ms)
- *
- * <p>File size: 24,872 KB Parallel run: 86 ms (124 ms, 79 ms, 56 ms) Files.readLines() run: 113 ms
- * (136 ms, 112 ms, 91 ms)
- *
- * <p>File size: 100,184 KB Parallel run: 74 ms (75 ms, 77 ms, 69 ms) Files.readLines() run: 418 ms
- * (540 ms, 360 ms, 354 ms)
- */
-@RunWith(JUnit4.class)
-public class ParallelFileProcessingTest {
- @Test
- public void testVariant1() throws Exception {
- int limit = 2000;
- int blockSize = -1;
- doTestNumbers(limit, blockSize);
- }
-
- @Test
- public void testVariant2() throws Exception {
- int limit = 4000;
- int blockSize = 100;
- doTestNumbers(limit, blockSize);
- }
-
- @Test
- public void testVariant3() throws Exception {
- int limit = 500;
- int blockSize = 1129;
- doTestNumbers(limit, blockSize);
- }
-
- @Test
- public void testPerformanceMedium() throws Exception {
- doPerformanceTest(randomFile(new Random(), 500));
- }
-
- private static void doPerformanceTest(File file) throws Exception {
- try {
- // Currently we do not call toString() method, as it reduces performance in X times;
- // However, further parsing / conversion to string can be done differently
- // (for instance, we could decode the underlying big byte buffers and read
- // corresponding decoded parts from there)
- long[] parallel =
- nTimesAvg(
- () -> {
- List<List<FileFragment>> list = Lists.newArrayList();
- Supplier<DeclarationConsumer> factory =
- () -> {
- List<FileFragment> inner = Lists.newArrayList();
- list.add(inner);
- return fragment -> inner.add(fragment);
- };
- parseFile(file, factory, null);
- assertThat(list).isNotEmpty();
- },
- 3);
- long[] usual =
- nTimesAvg(
- () -> {
- List<String> usualLines = Files.readLines(file, StandardCharsets.ISO_8859_1);
- assertThat(usualLines).isNotEmpty();
- },
- 3);
- printPerformanceResults(file, parallel, usual);
- } finally {
- file.delete();
- }
- }
-
- private static void parseFile(
- File file,
- Supplier<DeclarationConsumer> factory,
- @Nullable ParallelFileProcessing.BlockParameters parameters)
- throws IOException, GenericParsingException, InterruptedException {
- ListeningExecutorService service =
- MoreExecutors.listeningDecorator(
- Executors.newFixedThreadPool(
- 25,
- new ThreadFactoryBuilder()
- .setNameFormat(ParallelFileProcessingTest.class.getSimpleName() + "-%d")
- .build()));
- try (SeekableByteChannel channel = java.nio.file.Files.newByteChannel(file.toPath())) {
- ParallelFileProcessing.processFile(
- channel,
- parameters != null ? parameters : new BlockParameters(file.length()),
- factory,
- service);
- } finally {
- ExecutorUtil.interruptibleShutdown(service);
- }
- }
-
- private static void printPerformanceResults(File file, long[] parallel, long[] usual) {
- System.out.println(
- String.format("\nFile size: %,d KB ", file.length() / 1024)
- + "\nParallel run: "
- + printTimes(parallel)
- + "\nFiles.readLines() run: "
- + printTimes(usual));
- }
-
- private static String printTimes(long[] times) {
- List<String> values = Lists.newArrayList();
- double avg = 0;
- for (long time : times) {
- avg += ((double) time) / times.length;
- values.add(time + " ms");
- }
- return Math.round(avg) + " ms (" + String.join(", ", values) + ")";
- }
-
- private static long[] nTimesAvg(Callback r, int num) throws Exception {
- long[] result = new long[num];
- for (int i = 0; i < num; i++) {
- Stopwatch stopwatch = Stopwatch.createStarted();
- r.process();
- result[i] = stopwatch.elapsed().toMillis();
- }
- return result;
- }
-
- private static void doTestNumbers(int limit, int blockSize)
- throws IOException, GenericParsingException, InterruptedException {
- File file = writeTestFile(limit);
- try {
- List<String> lines = Collections.synchronizedList(Lists.newArrayListWithCapacity(limit));
- parseFile(
- file,
- () -> fragment -> lines.add(fragment.toString()),
- new BlockParameters(file.length()).setReadBlockSize(blockSize));
- // Copy to non-synchronized list for check
- assertNumbers(limit, Lists.newArrayList(lines));
- } finally {
- file.delete();
- }
- }
-
- private static void assertNumbers(int limit, List<String> lines) {
- Set<Integer> numbers = Sets.newHashSet();
- lines.forEach(
- s -> {
- boolean added = numbers.add(Integer.parseInt(s.trim()));
- assertThat(added).isTrue();
- });
- assertThat(numbers).hasSize(limit);
- for (int i = 0; i < limit; i++) {
- boolean removed = numbers.remove(i);
- assertThat(removed).isTrue();
- }
- }
-
- private static File randomFile(Random r, int limit) throws IOException {
- String[] strings = new String[limit];
- IntStream.range(0, limit)
- .parallel()
- .forEach(
- i -> {
- StringBuilder sb = new StringBuilder();
- int len = 100 + r.nextInt(10000);
- for (int j = 0; j < len; j++) {
- sb.append(r.nextInt());
- int value = r.nextInt(50);
- if (value == 5) {
- sb.append('\n');
- }
- }
-
- strings[i] = sb.toString();
- });
- File file = File.createTempFile("test", ".txt");
- Files.asCharSink(file, StandardCharsets.ISO_8859_1).write(String.join("\n", strings));
- return file;
- }
-
- private static File writeTestFile(int limit) throws IOException {
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < limit; i++) {
- if (sb.length() > 0) {
- sb.append('\n');
- }
- sb.append(i);
- }
- File file = File.createTempFile("test", ".txt");
- Files.asCharSink(file, StandardCharsets.ISO_8859_1).write(sb);
- return file;
- }
-
- private interface Callback {
- void process() throws Exception;
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/blackbox/tests/BUILD b/src/test/java/com/google/devtools/build/lib/blackbox/tests/BUILD
index c779489..7cfaaf3 100644
--- a/src/test/java/com/google/devtools/build/lib/blackbox/tests/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/blackbox/tests/BUILD
@@ -52,19 +52,6 @@
],
)
-java_test(
- name = "NinjaBlackBoxTest",
- timeout = "moderate",
- srcs = ["NinjaBlackBoxTest.java"],
- tags = [
- "black_box_test",
- "no_windows",
- ],
- deps = [
- ":common_deps",
- ],
-)
-
test_suite(
name = "black_box_tests",
tags = ["black_box_test"],
diff --git a/src/test/java/com/google/devtools/build/lib/blackbox/tests/NinjaBlackBoxTest.java b/src/test/java/com/google/devtools/build/lib/blackbox/tests/NinjaBlackBoxTest.java
deleted file mode 100644
index 10abca8..0000000
--- a/src/test/java/com/google/devtools/build/lib/blackbox/tests/NinjaBlackBoxTest.java
+++ /dev/null
@@ -1,704 +0,0 @@
-// Copyright 2020 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.blackbox.tests;
-
-import static com.google.common.truth.Truth.assertThat;
-import static org.junit.Assert.assertThrows;
-
-import com.google.devtools.build.lib.blackbox.framework.BuilderRunner;
-import com.google.devtools.build.lib.blackbox.framework.ProcessResult;
-import com.google.devtools.build.lib.blackbox.junit.AbstractBlackBoxTest;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.List;
-import org.junit.Before;
-import org.junit.Test;
-
-/** Integration test for Ninja execution functionality. */
-public class NinjaBlackBoxTest extends AbstractBlackBoxTest {
- @Override
- @Before
- public void setUp() throws Exception {
- super.setUp();
- context().write(".bazelignore", "build_dir");
- context()
- .write(
- WORKSPACE,
- String.format("workspace(name = '%s')", testName.getMethodName()),
- "toplevel_output_directories(paths = ['build_dir'])");
- }
-
- @Test
- public void testOneTarget() throws Exception {
- context().write("build_dir/input.txt", "World");
- context()
- .write(
- "build_dir/build.ninja",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- "build hello.txt: echo input.txt");
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build.ninja',",
- " output_root_inputs = ['input.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups = {'group': ['hello.txt']})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
- assertConfigured(bazel.build("//:ninja_target"));
- Path path = context().resolveExecRootPath(bazel, "build_dir/hello.txt");
- assertThat(path.toFile().exists()).isTrue();
- assertThat(Files.readAllLines(path)).containsExactly("Hello World!");
-
- // React to input change.
- context().write("build_dir/input.txt", "Sun");
- assertNothingConfigured(bazel.build("//:ninja_target"));
- assertThat(Files.readAllLines(path)).containsExactly("Hello Sun!");
-
- // React to Ninja file change.
- context()
- .write(
- "build_dir/build.ninja",
- "rule echo",
- " command = echo \"Hello $$(cat ${in}):)\" > ${out}",
- "build hello.txt: echo input.txt");
- assertConfigured(bazel.build("//:ninja_target"));
- assertThat(Files.readAllLines(path)).containsExactly("Hello Sun:)");
- }
-
- @Test
- public void testWithoutExperimentalFlag() throws Exception {
- context().write("build_dir/input.txt", "World");
- context()
- .write(
- "build_dir/build.ninja",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- "build hello.txt: echo input.txt");
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build.ninja',",
- " output_root_inputs = ['input.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups = {'group': ['hello.txt']})");
-
- BuilderRunner bazel = context().bazel();
- ProcessResult result = bazel.shouldFail().build("//:ninja_target");
- assertThat(result.errString()).contains("name 'toplevel_output_directories' is not defined");
- assertThat(result.errString()).contains("FAILED: Build did NOT complete successfully");
- }
-
- @Test
- public void testWithoutMainNinja() throws Exception {
- context().write("build_dir/input.txt", "World");
- context()
- .write(
- "build_dir/build.ninja",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- "build hello.txt: echo input.txt");
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " output_root_inputs = ['input.txt'])",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups = {'group': ['hello.txt']})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
- ProcessResult result = bazel.shouldFail().build("//:ninja_target");
- assertThat(result.errString())
- .contains("//:graph: missing value for mandatory attribute 'main' in 'ninja_graph' rule");
- assertThat(result.errString()).contains("FAILED: Build did NOT complete successfully");
- }
-
- @Test
- public void testSourceFileIsMissingUnderOutputRoot() throws Exception {
- context().write("input.txt", "World");
- context()
- .write(
- "build_dir/build.ninja",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- "build hello.txt: echo build_dir/input.txt");
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build.ninja')",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups = {'group': ['hello.txt']})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
- ProcessResult result = bazel.shouldFail().build("//:ninja_target");
- assertThat(result.errString())
- .contains(
- "in ninja_build rule //:ninja_target: The following artifacts do not have a generating "
- + "action in Ninja file: build_dir/build_dir/input.txt");
- assertThat(result.errString()).contains("FAILED: Build did NOT complete successfully");
- }
-
- private static void assertNothingConfigured(ProcessResult result) {
- assertThat(result.errString())
- .contains(
- "INFO: Analyzed target //:ninja_target (0 packages loaded, 0 targets configured).");
- }
-
- private static void assertConfigured(ProcessResult result) {
- assertThat(result.errString())
- .doesNotContain(
- "INFO: Analyzed target //:ninja_target (0 packages loaded, 0 targets configured).");
- }
-
- @Test
- public void testNullBuild() throws Exception {
- // Print nanoseconds fraction of the current time into the output file.
- context()
- .write(
- "build_dir/build.ninja",
- "rule echo_time",
- " command = date +%N >> ${out}",
- "build nano.txt: echo_time");
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', ",
- "output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build.ninja')",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups = {'group': ['nano.txt']})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
- assertConfigured(bazel.build("//:ninja_target"));
- Path path = context().resolveExecRootPath(bazel, "build_dir/nano.txt");
- assertThat(path.toFile().exists()).isTrue();
- List<String> text = Files.readAllLines(path);
- assertThat(text).isNotEmpty();
- long lastModified = path.toFile().lastModified();
-
- // Should be null build, as nothing changed.
- assertNothingConfigured(bazel.build("//:ninja_target"));
- assertThat(Files.readAllLines(path)).containsExactly(text.get(0));
- assertThat(path.toFile().lastModified()).isEqualTo(lastModified);
- }
-
- @Test
- public void testInteroperabilityWithBazel() throws Exception {
- context().write("bazel_input.txt", "World");
- context()
- .write(
- "build_dir/build.ninja",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- "build hello.txt: echo placeholder",
- "build hello2.txt: echo placeholder2");
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build.ninja')",
- "filegroup(name = 'bazel_built_input', srcs = [':bazel_input.txt'])",
- "ninja_build(name = 'ninja_target1', ninja_graph = 'graph',",
- " deps_mapping = {'placeholder': ':bazel_built_input'},",
- " output_groups = {'group': ['hello.txt']})",
- "filegroup(name = 'bazel_middle', srcs = [':ninja_target1'])",
- "ninja_build(name = 'ninja_target2', ninja_graph = 'graph',",
- " deps_mapping = {'placeholder2': ':bazel_middle'},",
- " output_groups = {'group': ['hello2.txt']})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
- assertConfigured(bazel.build("//..."));
- Path path1 = context().resolveExecRootPath(bazel, "build_dir/hello.txt");
- Path path2 = context().resolveExecRootPath(bazel, "build_dir/hello2.txt");
-
- assertThat(Files.readAllLines(path1)).containsExactly("Hello World!");
- assertThat(Files.readAllLines(path2)).containsExactly("Hello Hello World!!");
- }
-
- @Test
- public void testInteroperabilityWithBazelCycle() throws Exception {
- context().write("bazel_input.txt", "World");
- context()
- .write(
- "build_dir/build.ninja",
- "rule echo",
- " command = echo \"Hello $$(cat ${in})!\" > ${out}",
- "build hello.txt: echo placeholder",
- "build hello2.txt: echo placeholder2");
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build.ninja')",
-
- // Cycle here with bazel_middle.
- "filegroup(name = 'bazel_built_input', srcs = [':bazel_input.txt', ':bazel_middle'])",
- "ninja_build(name = 'ninja_target1', ninja_graph = 'graph',",
- " deps_mapping = {'placeholder': ':bazel_built_input'},",
- " output_groups = {'group': ['hello.txt']})",
- "filegroup(name = 'bazel_middle', srcs = [':ninja_target1'])",
- "ninja_build(name = 'ninja_target2', ninja_graph = 'graph',",
- " deps_mapping = {'placeholder2': ':bazel_middle'},",
- " output_groups = {'group': ['hello2.txt']})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
- Exception exception = assertThrows(Exception.class, () -> bazel.build("//..."));
- assertThat(exception).hasMessageThat().contains("cycle in dependency graph");
- }
-
- @Test
- public void testDisjointPhonyNinjaParts() throws Exception {
- context().write("build_dir/a.txt", "A");
- context().write("build_dir/b.txt", "B");
- context().write("build_dir/c.txt", "C");
- context().write("build_dir/d.txt", "D");
- context().write("build_dir/e.txt", "E");
-
- context()
- .write(
- "build_dir/build.ninja",
- "rule cat",
- " command = echo '<<' $$(cat ${in}) '>>' > ${out}",
- "rule echo",
- " command = echo \"Hello $$(cat ${in} | tr '\\r\\n' ' ')!\" > ${out}",
- "build a: cat a.txt",
- "build b: cat b.txt",
- "build c: cat c.txt",
- "build d: cat d.txt",
- "build e: cat e.txt",
- "build group1: phony a b c",
- "build group2: phony d e",
- "build inputs_alias: phony group1 group2",
- "build hello.txt: echo inputs_alias",
- "build alias: phony hello.txt");
-
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build.ninja',",
- " output_root_inputs = ['a.txt', 'b.txt', 'c.txt', 'd.txt', 'e.txt'])",
- "ninja_build(name = 'ninja_target1', ninja_graph = 'graph',",
- " output_groups= {'main': ['group1']})",
- "ninja_build(name = 'ninja_target2', ninja_graph = 'graph',",
- " output_groups= {'main': ['group2']})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
- assertConfigured(bazel.build("//..."));
- Path pathA = context().resolveExecRootPath(bazel, "build_dir/a");
- Path pathB = context().resolveExecRootPath(bazel, "build_dir/b");
- Path pathC = context().resolveExecRootPath(bazel, "build_dir/c");
- Path pathD = context().resolveExecRootPath(bazel, "build_dir/d");
- Path pathE = context().resolveExecRootPath(bazel, "build_dir/e");
-
- assertThat(Files.readAllLines(pathA)).containsExactly("<< A >>");
- assertThat(Files.readAllLines(pathB)).containsExactly("<< B >>");
- assertThat(Files.readAllLines(pathC)).containsExactly("<< C >>");
- assertThat(Files.readAllLines(pathD)).containsExactly("<< D >>");
- assertThat(Files.readAllLines(pathE)).containsExactly("<< E >>");
- }
-
- @Test
- public void testPhonyNinjaPartsWithSharedPart() throws Exception {
- context().write("build_dir/a.txt", "A");
- context().write("build_dir/b.txt", "B");
- context().write("build_dir/c.txt", "C");
- context().write("build_dir/d.txt", "D");
- context().write("build_dir/e.txt", "E");
-
- context()
- .write(
- "build_dir/build.ninja",
- "rule cat",
- " command = echo '<<' $$(cat ${in}) '>>' > ${out}",
- "build a: cat a.txt",
- "build b: cat b.txt",
- "build c: cat c.txt",
- "build d: cat d.txt",
- "build e: cat e.txt",
- // 'a' is present in both groups, built by Bazel since file 'a' is produced by
- // equal-without-owner actions.
- "build group1: phony a b c",
- "build group2: phony a d e");
-
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build.ninja',",
- " output_root_inputs = ['a.txt', 'b.txt', 'c.txt', 'd.txt', 'e.txt'])",
- "ninja_build(name = 'ninja_target1', ninja_graph = 'graph',",
- " output_groups= {'main': ['group1']})",
- "ninja_build(name = 'ninja_target2', ninja_graph = 'graph',",
- " output_groups= {'main': ['group2']})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
- assertConfigured(bazel.build("//..."));
- Path pathA = context().resolveExecRootPath(bazel, "build_dir/a");
- Path pathB = context().resolveExecRootPath(bazel, "build_dir/b");
- Path pathC = context().resolveExecRootPath(bazel, "build_dir/c");
- Path pathD = context().resolveExecRootPath(bazel, "build_dir/d");
- Path pathE = context().resolveExecRootPath(bazel, "build_dir/e");
-
- assertThat(Files.readAllLines(pathA)).containsExactly("<< A >>");
- assertThat(Files.readAllLines(pathB)).containsExactly("<< B >>");
- assertThat(Files.readAllLines(pathC)).containsExactly("<< C >>");
- assertThat(Files.readAllLines(pathD)).containsExactly("<< D >>");
- assertThat(Files.readAllLines(pathE)).containsExactly("<< E >>");
- }
-
- @Test
- public void testDisjointNinjaParts() throws Exception {
- context().write("build_dir/a.txt", "A");
- context().write("build_dir/b.txt", "B");
- context().write("build_dir/c.txt", "C");
- context().write("build_dir/d.txt", "D");
- context().write("build_dir/e.txt", "E");
-
- context()
- .write(
- "build_dir/build.ninja",
- "rule cat",
- " command = echo '<<' $$(cat ${in}) '>>' > ${out}",
- "build a: cat a.txt",
- "build b: cat b.txt",
- "build c: cat c.txt",
- "build d: cat d.txt",
- "build e: cat e.txt",
- "build group1: phony a b c",
- "build group2: phony d e",
- "build inputs_alias: phony group1 group2",
- "build hello.txt: cat inputs_alias",
- "build alias: phony hello.txt");
-
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build.ninja',",
- " output_root_inputs = ['a.txt', 'b.txt', 'c.txt', 'd.txt', 'e.txt'])",
- "ninja_build(name = 'ninja_target1', ninja_graph = 'graph',",
- " output_groups= {'main': ['a']})",
- "ninja_build(name = 'ninja_target2', ninja_graph = 'graph',",
- " output_groups= {'main': ['e']})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
- assertConfigured(bazel.build("//..."));
- Path pathA = context().resolveExecRootPath(bazel, "build_dir/a");
- Path pathE = context().resolveExecRootPath(bazel, "build_dir/e");
- assertThat(Files.readAllLines(pathA)).containsExactly("<< A >>");
- assertThat(Files.readAllLines(pathE)).containsExactly("<< E >>");
-
- Path pathB = context().resolveExecRootPath(bazel, "build_dir/b");
- Path pathC = context().resolveExecRootPath(bazel, "build_dir/c");
- Path pathD = context().resolveExecRootPath(bazel, "build_dir/d");
- assertThat(Files.exists(pathB)).isFalse();
- assertThat(Files.exists(pathC)).isFalse();
- assertThat(Files.exists(pathD)).isFalse();
- }
-
- @Test
- public void testDuplicateNinjaParts() throws Exception {
- context().write("build_dir/a.txt", "A");
- context().write("build_dir/b.txt", "B");
- context().write("build_dir/c.txt", "C");
- context().write("build_dir/d.txt", "D");
- context().write("build_dir/e.txt", "E");
-
- context()
- .write(
- "build_dir/build.ninja",
- "rule cat",
- " command = echo '<<' $$(cat ${in}) '>>' > ${out}",
- "build a: cat a.txt",
- "build b: cat b.txt");
-
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build.ninja',",
- " output_root_inputs = ['a.txt', 'b.txt', 'c.txt', 'd.txt', 'e.txt'])",
- // 'a' is present in both ninja_build targets, built by Bazel since file 'a' is produced
- // by equal-without-owner actions.
- "ninja_build(name = 'ninja_target1', ninja_graph = 'graph',",
- " output_groups= {'main': ['a']})",
- "ninja_build(name = 'ninja_target2', ninja_graph = 'graph',",
- " output_groups= {'main': ['a']})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
- assertConfigured(bazel.build("//..."));
- Path pathA = context().resolveExecRootPath(bazel, "build_dir/a");
- assertThat(Files.readAllLines(pathA)).containsExactly("<< A >>");
-
- Path pathB = context().resolveExecRootPath(bazel, "build_dir/b");
- assertThat(Files.exists(pathB)).isFalse();
- }
-
- @Test
- public void testDuplicateNinjaPartsDifferentMappings() throws Exception {
- context().write("variant1.txt", "variant1");
- context().write("variant2.txt", "variant2");
-
- context()
- .write(
- "build_dir/build.ninja",
- "rule append",
- " command = echo '<<' $$(cat ${in}) '>>' >> ${out}",
- "build a: append a.txt");
-
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build.ninja')",
- "ninja_build(name = 'ninja_target1', ninja_graph = 'graph',",
- " output_groups= {'main': ['a']}, deps_mapping = {'a.txt': ':variant1.txt'})",
- "ninja_build(name = 'ninja_target2', ninja_graph = 'graph',",
- " output_groups= {'main': ['a']}, deps_mapping = {'a.txt': ':variant2.txt'})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
-
- Exception exception = assertThrows(Exception.class, () -> bazel.build("//..."));
- assertThat(exception)
- .hasMessageThat()
- .contains("ERROR: file 'a.txt' is generated by these conflicting actions:");
- assertThat(exception)
- .hasMessageThat()
- .containsMatch(
- "for a\\.txt, previous action: action "
- + "'Symlinking deps_mapping entry 'variant[12]\\.txt' to 'build_dir/a\\.txt'', "
- + "attempted action: action "
- + "'Symlinking deps_mapping entry 'variant[12]\\.txt' to 'build_dir/a\\.txt''");
- }
-
- @Test
- public void testDependentNinjaActions() throws Exception {
- context().write("build_dir/a.txt", "A");
-
- context()
- .write(
- "build_dir/build1.ninja",
- "rule cat",
- " command = echo '<<' $$(cat ${in}) '>>' > ${out}",
- "build first.txt: cat a.txt");
- context()
- .write(
- "build_dir/build2.ninja",
- "rule cat",
- " command = echo '<<' $$(cat ${in}) '>>' > ${out}",
- "build second.txt: cat input");
-
- // For the dependent Ninja actions from the same Ninja graph, Ninja mechanisms should be used.
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph1', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build1.ninja',",
- " output_root_inputs = ['a.txt'])",
- "ninja_build(name = 'ninja_target1', ninja_graph = 'graph1',",
- " output_groups= {'main': ['first.txt']})",
- "ninja_graph(name = 'graph2', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build2.ninja')",
- "ninja_build(name = 'ninja_target2', ninja_graph = 'graph2',",
- " output_groups= {'main': ['second.txt']}, deps_mapping = {'input':"
- + " ':ninja_target1'})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
- assertConfigured(bazel.build("//..."));
- Path pathFirst = context().resolveExecRootPath(bazel, "build_dir/first.txt");
- assertThat(Files.readAllLines(pathFirst)).containsExactly("<< A >>");
- Path pathSecond = context().resolveExecRootPath(bazel, "build_dir/second.txt");
- assertThat(Files.readAllLines(pathSecond)).containsExactly("<< << A >> >>");
- }
-
- @Test
- public void testDependentNinjaActionsCycle() throws Exception {
- context()
- .write(
- "build_dir/build1.ninja",
- "rule cat",
- " command = echo '<<' $$(cat ${in}) '>>' > ${out}",
- "build first.txt: cat input");
- context()
- .write(
- "build_dir/build2.ninja",
- "rule cat",
- " command = echo '<<' $$(cat ${in}) '>>' > ${out}",
- "build second.txt: cat input");
-
- // For the dependent Ninja actions from the same Ninja graph, Ninja mechanisms should be used.
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph1', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build1.ninja')",
- "ninja_build(name = 'ninja_target1', ninja_graph = 'graph1',",
- " output_groups= {'main': ['first.txt']}, deps_mapping = {'input': ':ninja_target2'})",
- "ninja_graph(name = 'graph2', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build2.ninja')",
- "ninja_build(name = 'ninja_target2', ninja_graph = 'graph2',",
- " output_groups= {'main': ['second.txt']}, deps_mapping = {'input':"
- + " ':ninja_target1'})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
- Exception exception = assertThrows(Exception.class, () -> bazel.build("//..."));
- assertThat(exception).hasMessageThat().contains("cycle in dependency graph");
- }
-
- @Test
- public void testRspFileWritten() throws Exception {
- context().write("input.txt", "input");
- context()
- .write(
- "build_dir/build.ninja",
- "rule cat",
- " command = echo '<<' $$(cat ${in}) '>>' > ${out}",
- " rspfile = ${out}.rsp",
- " rspfile_content = ${in}",
- "build first.txt: cat ../input.txt");
-
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build.ninja')",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups= {'main': ['first.txt']})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
- assertConfigured(bazel.build("//..."));
- Path pathFirst = context().resolveExecRootPath(bazel, "build_dir/first.txt");
- assertThat(Files.readAllLines(pathFirst)).containsExactly("<< input >>");
-
- Path rspFile = context().resolveExecRootPath(bazel, "build_dir/first.txt.rsp");
- assertThat(Files.exists(rspFile)).isTrue();
- assertThat(Files.readAllLines(rspFile)).containsExactly("../input.txt");
- }
-
- @Test
- public void testOrderOnlyInputsDoNotCauseRebuild() throws Exception {
- context().write("input.txt", "abc");
- context().write("order-only-input.txt", "123");
- context()
- .write(
- "build_dir/build.ninja",
- "rule cat",
- " command = cat ${in} ${order_only_input} > ${out}",
- "build output.txt: cat ../input.txt || ../order-only-input.txt",
- " order_only_input = ../order-only-input.txt");
-
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build.ninja')",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups = {'main': ['output.txt']})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
-
- assertConfigured(bazel.build("//:ninja_target"));
- Path outputPath = context().resolveExecRootPath(bazel, "build_dir/output.txt");
- assertThat(outputPath.toFile().exists()).isTrue();
- assertThat(Files.readAllLines(outputPath)).containsExactly("abc", "123");
-
- // Change the order-only input...
- context().write("order-only-input.txt", "456");
- // ...but nothing should rerun....
- assertNothingConfigured(bazel.build("//:ninja_target"));
- // ...and the output should remain the same.
- assertThat(Files.readAllLines(outputPath)).containsExactly("abc", "123");
-
- // Change the main input...
- context().write("input.txt", "xyz");
- bazel.build("//:ninja_target");
- // ...and the output should have the new content from the order-only input.
- assertThat(Files.readAllLines(outputPath)).containsExactly("xyz", "456");
- }
-
- @Test
- public void testOrderOnlyInputsInDepFileCauseRebuild() throws Exception {
- context().write("build_dir/in.txt", "abc");
- context().write("build_dir/order-only-input1.txt", "123");
- context().write("build_dir/order-only-input2.txt", "456");
- context()
- .write(
- "build_dir/build.ninja",
- "rule cat",
- " command = cat ${in} ${order_only_input1} ${order_only_input2} > ${out} ; "
- // TODO(b/156961649): Having to put "build_dir/" in front of ${order_only_input2}
- // might be a bug
- + "echo \"${out} : build_dir/${order_only_input2}\" > ${depfile}",
- " depfile = df",
- "build output.txt: cat in.txt || order-only-input1.txt order-only-input2.txt",
- " order_only_input1 = order-only-input1.txt",
- " order_only_input2 = order-only-input2.txt");
-
- context()
- .write(
- "BUILD",
- "ninja_graph(name = 'graph', output_root = 'build_dir',",
- " working_directory = 'build_dir',",
- " main = 'build_dir/build.ninja',",
- " output_root_inputs = ['in.txt', 'order-only-input1.txt', 'order-only-input2.txt'],)",
- "ninja_build(name = 'ninja_target', ninja_graph = 'graph',",
- " output_groups = {'main': ['output.txt']})");
-
- BuilderRunner bazel = context().bazel().withFlags("--experimental_ninja_actions");
-
- ProcessResult pr = bazel.build("//:ninja_target");
- System.out.println(pr.outString());
- assertConfigured(pr);
- Path outputPath = context().resolveExecRootPath(bazel, "build_dir/output.txt");
- assertThat(outputPath.toFile().exists()).isTrue();
- assertThat(Files.readAllLines(outputPath)).containsExactly("abc", "123", "456");
-
- // Change the order-only input...
- context().write("build_dir/order-only-input1.txt", "789");
- // ...but nothing should rerun....
- assertNothingConfigured(bazel.build("//:ninja_target"));
- // ...and the output should remain the same.
- assertThat(Files.readAllLines(outputPath)).containsExactly("abc", "123", "456");
-
- // But change the order-only input that's listed in the depfile...
- context().write("build_dir/order-only-input2.txt", "xyz");
- bazel.build("//:ninja_target");
- // ...and the output should have the new content from both order-only inputs.
- assertThat(Files.readAllLines(outputPath)).containsExactly("abc", "789", "xyz");
- }
-}
diff --git a/src/test/java/com/google/devtools/build/lib/buildtool/SymlinkForestTest.java b/src/test/java/com/google/devtools/build/lib/buildtool/SymlinkForestTest.java
index e071655..1c54091 100644
--- a/src/test/java/com/google/devtools/build/lib/buildtool/SymlinkForestTest.java
+++ b/src/test/java/com/google/devtools/build/lib/buildtool/SymlinkForestTest.java
@@ -16,19 +16,16 @@
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
-import static org.junit.Assert.assertThrows;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSortedSet;
import com.google.devtools.build.lib.cmdline.LabelConstants;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.testutil.ManualClock;
import com.google.devtools.build.lib.testutil.TestConstants;
-import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
@@ -38,7 +35,6 @@
import com.google.devtools.build.lib.vfs.Symlinks;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -138,8 +134,7 @@
@Test
public void testDeleteTreesBelowNotPrefixed() throws IOException {
createTestDirectoryTree();
- SymlinkForest.deleteTreesBelowNotPrefixed(
- topDir, "file-", /*notSymlinkedInExecrootDirectories=*/ ImmutableSortedSet.of());
+ SymlinkForest.deleteTreesBelowNotPrefixed(topDir, "file-");
assertThat(file1.exists()).isTrue();
assertThat(file2.exists()).isTrue();
assertThat(aDir.exists()).isFalse();
@@ -223,12 +218,7 @@
Path linkRoot = fileSystem.getPath("/linkRoot");
linkRoot.createDirectoryAndParents();
ImmutableList<Path> plantedSymlinks =
- new SymlinkForest(
- packageRootMap,
- linkRoot,
- TestConstants.PRODUCT_NAME,
- ImmutableSortedSet.of(),
- false)
+ new SymlinkForest(packageRootMap, linkRoot, TestConstants.PRODUCT_NAME, false)
.plantSymlinkForest();
assertLinksTo(linkRoot, rootA, "pkgA");
@@ -268,12 +258,7 @@
.buildOrThrow();
ImmutableList<Path> plantedSymlinks =
- new SymlinkForest(
- packageRootMap,
- linkRoot,
- TestConstants.PRODUCT_NAME,
- ImmutableSortedSet.of(),
- false)
+ new SymlinkForest(packageRootMap, linkRoot, TestConstants.PRODUCT_NAME, false)
.plantSymlinkForest();
assertLinksTo(linkRoot, rootX, "file");
assertThat(plantedSymlinks)
@@ -308,12 +293,7 @@
.build();
ImmutableList<Path> plantedSymlinks =
- new SymlinkForest(
- packageRootMap,
- linkRoot,
- TestConstants.PRODUCT_NAME,
- ImmutableSortedSet.of(),
- false)
+ new SymlinkForest(packageRootMap, linkRoot, TestConstants.PRODUCT_NAME, false)
.plantSymlinkForest();
assertLinksTo(linkRoot, mainRepo, "dir_main");
@@ -372,8 +352,7 @@
.build();
ImmutableList<Path> plantedSymlinks =
- new SymlinkForest(
- packageRootMap, linkRoot, TestConstants.PRODUCT_NAME, ImmutableSortedSet.of(), true)
+ new SymlinkForest(packageRootMap, linkRoot, TestConstants.PRODUCT_NAME, true)
.plantSymlinkForest();
// Expected sibling repository layout (X, Y and Z are siblings of ws_name):
@@ -437,12 +416,7 @@
.buildOrThrow();
ImmutableList<Path> plantedSymlinks =
- new SymlinkForest(
- packageRootMap,
- linkRoot,
- TestConstants.PRODUCT_NAME,
- ImmutableSortedSet.of(),
- false)
+ new SymlinkForest(packageRootMap, linkRoot, TestConstants.PRODUCT_NAME, false)
.plantSymlinkForest();
assertLinksTo(linkRoot, mainRepo, "dir1");
@@ -484,12 +458,7 @@
.build();
ImmutableList<Path> plantedSymlinks =
- new SymlinkForest(
- packageRootMap,
- linkRoot,
- TestConstants.PRODUCT_NAME,
- ImmutableSortedSet.of(),
- false)
+ new SymlinkForest(packageRootMap, linkRoot, TestConstants.PRODUCT_NAME, false)
.plantSymlinkForest();
assertLinksTo(linkRoot, mainRepo, "dir1");
@@ -529,12 +498,7 @@
.build();
ImmutableList<Path> plantedSymlinks =
- new SymlinkForest(
- packageRootMap,
- linkRoot,
- TestConstants.PRODUCT_NAME,
- ImmutableSortedSet.of(),
- false)
+ new SymlinkForest(packageRootMap, linkRoot, TestConstants.PRODUCT_NAME, false)
.plantSymlinkForest();
assertLinksTo(linkRoot, mainRepo, "dir1");
@@ -574,8 +538,7 @@
.build();
ImmutableList<Path> plantedSymlinks =
- new SymlinkForest(
- packageRootMap, linkRoot, TestConstants.PRODUCT_NAME, ImmutableSortedSet.of(), true)
+ new SymlinkForest(packageRootMap, linkRoot, TestConstants.PRODUCT_NAME, true)
.plantSymlinkForest();
// Expected output base layout with sibling repositories in the execroot where
@@ -643,8 +606,7 @@
.buildOrThrow();
ImmutableList<Path> plantedSymlinks =
- new SymlinkForest(
- packageRootMap, linkRoot, TestConstants.PRODUCT_NAME, ImmutableSortedSet.of(), true)
+ new SymlinkForest(packageRootMap, linkRoot, TestConstants.PRODUCT_NAME, true)
.plantSymlinkForest();
// Expected output base layout with sibling repositories in the execroot where
@@ -692,217 +654,9 @@
.build();
ImmutableList<Path> plantedSymlinks =
- new SymlinkForest(
- packageRootMap,
- linkRoot,
- TestConstants.PRODUCT_NAME,
- ImmutableSortedSet.of(),
- false)
+ new SymlinkForest(packageRootMap, linkRoot, TestConstants.PRODUCT_NAME, false)
.plantSymlinkForest();
assertThat(linkRoot.getRelative(LabelConstants.EXTERNAL_PATH_PREFIX).exists()).isFalse();
assertThat(plantedSymlinks).isEmpty();
}
-
- @Test
- public void testNotSymlinkedDirectoriesInExecRootAllInMainRepo() throws Exception {
- Root outputBase = Root.fromPath(fileSystem.getPath("/ob"));
- Root mainRepo = Root.fromPath(fileSystem.getPath("/my_repo"));
- Path linkRoot = outputBase.getRelative("execroot/ws_name");
-
- linkRoot.createDirectoryAndParents();
- mainRepo.asPath().createDirectoryAndParents();
- mainRepo.getRelative("dir3").createDirectoryAndParents();
- mainRepo.getRelative("build").createDirectoryAndParents();
-
- ImmutableMap<PackageIdentifier, Root> packageRootMap =
- ImmutableMap.<PackageIdentifier, Root>builder()
- .put(createMainPkg(mainRepo, "dir1/pkg/foo"), mainRepo)
- .put(createMainPkg(mainRepo, "dir2/pkg"), mainRepo)
- // Empty package will cause every top-level files to be linked, except external/
- .put(createMainPkg(mainRepo, ""), mainRepo)
- .put(
- createExternalPkg(outputBase, "X", "dir_x/pkg"),
- externalSourceRoot(outputBase, "X"))
- .build();
-
- ImmutableList<Path> plantedSymlinks =
- new SymlinkForest(
- packageRootMap,
- linkRoot,
- TestConstants.PRODUCT_NAME,
- ImmutableSortedSet.of("build"),
- false)
- .plantSymlinkForest();
-
- assertLinksTo(linkRoot, mainRepo, "dir1");
- assertLinksTo(linkRoot, mainRepo, "dir2");
- assertLinksTo(linkRoot, mainRepo, "dir3");
- assertLinksToExternalRepo(linkRoot, outputBase, "X");
- assertThat(linkRoot.getChild("build").exists()).isFalse();
- assertThat(plantedSymlinks)
- .containsExactly(
- linkRoot.getRelative("file"),
- linkRoot.getRelative("dir1"),
- linkRoot.getRelative("dir2"),
- linkRoot.getRelative("dir3"),
- linkRoot.getRelative(LabelConstants.EXTERNAL_PATH_PREFIX + "/X"));
- }
-
- @Test
- public void testNotSymlinkedDirectoriesNotDeletedBetweenCommands() throws Exception {
- Root outputBase = Root.fromPath(fileSystem.getPath("/ob"));
- Root mainRepo = Root.fromPath(fileSystem.getPath("/my_repo"));
- Path linkRoot = outputBase.getRelative("execroot/ws_name");
-
- linkRoot.createDirectoryAndParents();
- mainRepo.asPath().createDirectoryAndParents();
- mainRepo.getRelative("build").createDirectoryAndParents();
-
- ImmutableMap<PackageIdentifier, Root> packageRootMap =
- ImmutableMap.<PackageIdentifier, Root>builder()
- .put(createMainPkg(mainRepo, "dir1/pkg"), mainRepo)
- // Empty package will cause every top-level files to be linked, except external/
- .put(createMainPkg(mainRepo, ""), mainRepo)
- .build();
-
- SymlinkForest symlinkForest =
- new SymlinkForest(
- packageRootMap,
- linkRoot,
- TestConstants.PRODUCT_NAME,
- ImmutableSortedSet.of("build"),
- false);
- symlinkForest.plantSymlinkForest();
-
- assertLinksTo(linkRoot, mainRepo, "dir1");
- assertThat(linkRoot.getChild("build").exists()).isFalse();
-
- // Create some file in 'build' directory under exec root.
- Path notSymlinkedDir = linkRoot.getChild("build");
- notSymlinkedDir.createDirectoryAndParents();
-
- byte[] bytes = "text".getBytes(StandardCharsets.ISO_8859_1);
- Path childPath = notSymlinkedDir.getChild("child.txt");
- FileSystemUtils.writeContent(childPath, bytes);
-
- symlinkForest.plantSymlinkForest();
-
- assertLinksTo(linkRoot, mainRepo, "dir1");
- // Exists because it was explicitly created.
- assertThat(linkRoot.getChild("build").exists()).isTrue();
- // The presence of the manually added file indicates that SymlinkForest did not delete
- // the directory it's in.
- assertThat(childPath.exists()).isTrue();
- assertThat(FileSystemUtils.readContent(childPath, StandardCharsets.ISO_8859_1))
- .isEqualTo("text");
- }
-
- @Test
- public void testNotSymlinkedDirectoriesInExecRootPartialMainRepo1() throws Exception {
- Root outputBase = Root.fromPath(fileSystem.getPath("/ob"));
- Root mainRepo = Root.fromPath(fileSystem.getPath("/my_repo"));
- Path linkRoot = outputBase.getRelative("execroot/ws_name");
-
- linkRoot.createDirectoryAndParents();
- mainRepo.asPath().createDirectoryAndParents();
- mainRepo.getRelative("dir3").createDirectoryAndParents();
- mainRepo.getRelative("build").createDirectoryAndParents();
-
- ImmutableMap<PackageIdentifier, Root> packageRootMap =
- ImmutableMap.<PackageIdentifier, Root>builder()
- .put(createMainPkg(mainRepo, "dir1/pkg/foo"), mainRepo)
- .put(createMainPkg(mainRepo, "dir2/pkg"), mainRepo)
- .put(
- createExternalPkg(outputBase, "X", "dir_x/pkg"),
- externalSourceRoot(outputBase, "X"))
- .buildOrThrow();
-
- ImmutableList<Path> plantedSymlinks =
- new SymlinkForest(
- packageRootMap,
- linkRoot,
- TestConstants.PRODUCT_NAME,
- ImmutableSortedSet.of("build"),
- false)
- .plantSymlinkForest();
-
- assertLinksTo(linkRoot, mainRepo, "dir1");
- assertLinksTo(linkRoot, mainRepo, "dir2");
- assertLinksToExternalRepo(linkRoot, outputBase, "X");
- assertThat(linkRoot.getChild("build").exists()).isFalse();
- // Not part of the package roots.
- assertThat(linkRoot.getChild("dir3").exists()).isFalse();
- assertThat(plantedSymlinks)
- .containsExactly(
- linkRoot.getRelative("dir1"),
- linkRoot.getRelative("dir2"),
- linkRoot.getRelative(LabelConstants.EXTERNAL_PATH_PREFIX + "/X"));
- }
-
- @Test
- public void testNotSymlinkedDirectoriesInExecRootPartialMainRepo2() throws Exception {
- Root outputBase = Root.fromPath(fileSystem.getPath("/ob"));
- Root mainRepo = Root.fromPath(fileSystem.getPath("/my_repo"));
- Path linkRoot = outputBase.getRelative("execroot/ws_name");
-
- linkRoot.createDirectoryAndParents();
- mainRepo.asPath().createDirectoryAndParents();
-
- ImmutableMap<PackageIdentifier, Root> packageRootMap =
- ImmutableMap.of(createMainPkg(mainRepo, "build"), mainRepo);
-
- AbruptExitException exception =
- assertThrows(
- AbruptExitException.class,
- () ->
- new SymlinkForest(
- packageRootMap,
- linkRoot,
- TestConstants.PRODUCT_NAME,
- ImmutableSortedSet.of("build"),
- false)
- .plantSymlinkForest());
- assertThat(exception)
- .hasMessageThat()
- .isEqualTo(
- "Directories specified with toplevel_output_directories should be "
- + "ignored and can not be used as sources.");
- }
-
- @Test
- public void testNotSymlinkedDirectoriesInExecRootMultiplePackageRoots() throws Exception {
- Root outputBase = Root.fromPath(fileSystem.getPath("/ob"));
- Root mainRepo = Root.fromPath(fileSystem.getPath("/my_repo"));
- Root otherRepo = Root.fromPath(fileSystem.getPath("/other_repo"));
- Path linkRoot = outputBase.getRelative("execroot/ws_name");
-
- mainRepo.getRelative("build").createDirectoryAndParents();
-
- linkRoot.createDirectoryAndParents();
- mainRepo.asPath().createDirectoryAndParents();
- otherRepo.asPath().createDirectoryAndParents();
-
- ImmutableMap<PackageIdentifier, Root> packageRootMap =
- ImmutableMap.<PackageIdentifier, Root>builder()
- .put(createMainPkg(mainRepo, "dir1"), mainRepo)
- .put(createMainPkg(otherRepo, "dir2"), otherRepo)
- .buildOrThrow();
-
- AbruptExitException exception =
- assertThrows(
- AbruptExitException.class,
- () ->
- new SymlinkForest(
- packageRootMap,
- linkRoot,
- TestConstants.PRODUCT_NAME,
- ImmutableSortedSet.of("build"),
- false)
- .plantSymlinkForest());
- assertThat(exception)
- .hasMessageThat()
- .isEqualTo(
- "toplevel_output_directories is not supported together "
- + "with --package_path option.");
- }
}
diff --git a/src/test/java/com/google/devtools/build/lib/packages/semantics/ConsistencyTest.java b/src/test/java/com/google/devtools/build/lib/packages/semantics/ConsistencyTest.java
index 10e62bf..bff2d6f 100644
--- a/src/test/java/com/google/devtools/build/lib/packages/semantics/ConsistencyTest.java
+++ b/src/test/java/com/google/devtools/build/lib/packages/semantics/ConsistencyTest.java
@@ -125,7 +125,6 @@
"--experimental_builtins_dummy=" + rand.nextBoolean(),
"--experimental_enable_android_migration_apis=" + rand.nextBoolean(),
"--experimental_google_legacy_api=" + rand.nextBoolean(),
- "--experimental_ninja_actions=" + rand.nextBoolean(),
"--experimental_platforms_api=" + rand.nextBoolean(),
"--incompatible_allow_tags_propagation=" + rand.nextBoolean(), // flag, Java names differ
"--experimental_cc_shared_library=" + rand.nextBoolean(),
@@ -167,7 +166,6 @@
.setBool(
BuildLanguageOptions.EXPERIMENTAL_ENABLE_ANDROID_MIGRATION_APIS, rand.nextBoolean())
.setBool(BuildLanguageOptions.EXPERIMENTAL_GOOGLE_LEGACY_API, rand.nextBoolean())
- .setBool(BuildLanguageOptions.EXPERIMENTAL_NINJA_ACTIONS, rand.nextBoolean())
.setBool(BuildLanguageOptions.EXPERIMENTAL_PLATFORMS_API, rand.nextBoolean())
.setBool(BuildLanguageOptions.EXPERIMENTAL_ALLOW_TAGS_PROPAGATION, rand.nextBoolean())
.setBool(BuildLanguageOptions.EXPERIMENTAL_CC_SHARED_LIBRARY, rand.nextBoolean())
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunctionTest.java
index ae1d3d6..bdc4c49 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunctionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunctionTest.java
@@ -32,7 +32,6 @@
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.packages.WorkspaceFileValue;
import com.google.devtools.build.lib.packages.WorkspaceFileValue.WorkspaceFileKey;
-import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
import com.google.devtools.build.lib.rules.repository.ManagedDirectoriesKnowledgeImpl;
import com.google.devtools.build.lib.rules.repository.ManagedDirectoriesKnowledgeImpl.ManagedDirectoriesListener;
import com.google.devtools.build.lib.skyframe.util.SkyframeExecutorTestUtils;
@@ -487,59 +486,6 @@
}
@Test
- public void testDoNotSymlinkInExecroot() throws Exception {
- StarlarkSemantics semantics = getStarlarkSemantics();
- Injectable injectable = getSkyframeExecutor().injectable();
-
- try {
- StarlarkSemantics semanticsWithNinjaActions =
- StarlarkSemantics.builder()
- .setBool(BuildLanguageOptions.EXPERIMENTAL_NINJA_ACTIONS, true)
- .build();
- PrecomputedValue.STARLARK_SEMANTICS.set(injectable, semanticsWithNinjaActions);
-
- assertThat(
- parseWorkspaceFileValue("toplevel_output_directories(paths = [\"out\"])")
- .getDoNotSymlinkInExecrootPaths())
- .containsExactly("out");
- assertThat(
- parseWorkspaceFileValue(
- "toplevel_output_directories(paths = [\"out\", \"one more with"
- + " space\"])")
- .getDoNotSymlinkInExecrootPaths())
- .containsExactly("out", "one more with space");
- // Empty sequence is allowed.
- assertThat(
- parseWorkspaceFileValue("toplevel_output_directories(paths = [])")
- .getDoNotSymlinkInExecrootPaths())
- .isEmpty();
-
- // Test now intentionally introduces errors.
- reporter.removeHandler(failFastHandler);
-
- parseWorkspaceFileValueWithError(
- "toplevel_output_directories should not "
- + "contain duplicate values: \"out\" is specified more then once.",
- "toplevel_output_directories(paths = [\"out\", \"out\"])");
- parseWorkspaceFileValueWithError(
- "toplevel_output_directories can only accept "
- + "top level directories under workspace, \"out/subdir\" "
- + "can not be specified as an attribute.",
- "toplevel_output_directories(paths = [\"out/subdir\"])");
- parseWorkspaceFileValueWithError(
- "Empty path can not be passed to " + "toplevel_output_directories.",
- "toplevel_output_directories(paths = [\"\"])");
- parseWorkspaceFileValueWithError(
- "toplevel_output_directories can only "
- + "accept top level directories under workspace, \"/usr/local/bin\" "
- + "can not be specified as an attribute.",
- "toplevel_output_directories(paths = [\"/usr/local/bin\"])");
- } finally {
- PrecomputedValue.STARLARK_SEMANTICS.set(injectable, semantics);
- }
- }
-
- @Test
public void testMangledExternalWorkspaceFileIsIgnored() throws Exception {
scratch.file("secondary/WORKSPACE", "garbage");
RootedPath workspace =
diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
index a72684d..27f08d5 100644
--- a/src/test/shell/bazel/BUILD
+++ b/src/test/shell/bazel/BUILD
@@ -777,14 +777,6 @@
)
sh_test(
- name = "ninja_build_test",
- size = "large",
- srcs = ["ninja_build_test.sh"],
- data = [":test-deps"],
- tags = ["no_windows"],
-)
-
-sh_test(
name = "tags_propagation_starlark_test",
size = "large",
srcs = ["tags_propagation_starlark_test.sh"],
diff --git a/src/test/shell/bazel/ninja_build_test.sh b/src/test/shell/bazel/ninja_build_test.sh
deleted file mode 100755
index 36ee278..0000000
--- a/src/test/shell/bazel/ninja_build_test.sh
+++ /dev/null
@@ -1,541 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2020 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Tests ninja_build build rule.
-
-# Load the test setup defined in the parent directory
-CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source "${CURRENT_DIR}/../integration_test_setup.sh" \
- || { echo "integration_test_setup.sh not found!" >&2; exit 1; }
-
-function setup_basic_ninja_build() {
- cat > WORKSPACE <<'EOF'
-toplevel_output_directories(paths = ["out"])
-EOF
-
-mkdir -p test
- cat > test/BUILD <<'EOF'
-
-ninja_graph(
- name = "ninjagraph",
- main = "build.ninja",
- output_root = "out",
-)
-
-ninja_build(
- name = "ninjabuild",
- ninja_graph = "ninjagraph",
- output_groups = {
- "out" : ["out/test/output.txt"],
- }
-)
-EOF
-
- cat > test/build.ninja <<'EOF'
-rule cattool
- depfile = out/test/depfile.d
- command = ${in} ${out}
-build out/test/output.txt: cattool test/cattool.sh
-EOF
-
- cat > test/cattool.sh <<'EOF'
-OUTPUT=${!#}
-DEPFILE="$(dirname $OUTPUT)/depfile.d"
-
-cat test/{one,two} > $OUTPUT
-
-echo "$1 : test/one test/two" > $DEPFILE
-EOF
- chmod +x test/cattool.sh
- printf "a" > test/one
- echo "b" > test/two
-
- bazel clean
-}
-
-# Test that the depfile specified in the ninja file is used to determine
-# action inputs (and thus ninja actions are incrementally correct with
-# respect to discovered dependencies).
-function test_basic_depfile_processing() {
- setup_basic_ninja_build
-
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
-
- cat bazel-workspace/out/test/output.txt &> $TEST_log
- expect_log "ab"
-
- printf "HELLO" > test/one
-
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
-
- cat bazel-workspace/out/test/output.txt &> $TEST_log
- expect_log "HELLOb"
-}
-
-function test_null_build() {
- setup_basic_ninja_build
-
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
- expect_log "INFO: 2 processes: 1 internal, 1 local"
-
- # Verify null build with hot server.
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
- expect_log "INFO: 1 process: 1 internal"
-
- bazel shutdown
-
- # Verify null build even after restart.
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
- expect_log "INFO: 1 process: 1 internal"
-}
-
-# Tests that newly discovered dependencies cause a rebuild after restart.
-function test_rebuild_discovered_deps_after_restart() {
- setup_basic_ninja_build
-
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
-
- cat bazel-workspace/out/test/output.txt &> $TEST_log
- expect_log "ab"
-
- bazel shutdown
- printf "HELLO" > test/one
-
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
-
- cat bazel-workspace/out/test/output.txt &> $TEST_log
- expect_log "HELLOb"
-}
-
-# Tests that newly discovered dependencies cause a rebuild after restart.
-function test_depfile_existing_dependencies() {
- setup_basic_ninja_build
-
- # Override ninja file to make test/one and test/two explicit dependencies.
- cat > test/build.ninja <<'EOF'
-rule cattool
- depfile = out/test/depfile.d
- command = ${in} ${out}
-build out/test/output.txt: cattool test/cattool.sh test/one test/two
-EOF
-
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
-
- cat bazel-workspace/out/test/output.txt &> $TEST_log
- expect_log "ab"
-
- printf "HELLO" > test/one
-
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
-
- cat bazel-workspace/out/test/output.txt &> $TEST_log
- expect_log "HELLOb"
-}
-
-# Tests Bazel behaves appropriately when depfile output is malformed.
-function test_depfile_junk() {
- setup_basic_ninja_build
- cat > test/cattool.sh <<'EOF'
-OUTPUT=${!#}
-DEPFILE="$(dirname $OUTPUT)/depfile.d"
-
-cat test/{one,two} > $OUTPUT
-
-echo "Haha this depfile is a bunch of garbage" > $DEPFILE
-EOF
-
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
-
- cat bazel-workspace/out/test/output.txt &> $TEST_log
- expect_log "ab"
-}
-
-# Tests Bazel behaves appropriately when depfiles contain invalid files.
-function test_depfile_invalid_file() {
- setup_basic_ninja_build
- cat > test/cattool.sh <<'EOF'
-OUTPUT=${!#}
-DEPFILE="$(dirname $OUTPUT)/depfile.d"
-
-cat test/{one,two} > $OUTPUT
-
-echo "$1 : test/one test/two test/doesntexist" > $DEPFILE
-EOF
-
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
-
- cat bazel-workspace/out/test/output.txt &> $TEST_log
- expect_log "ab"
-}
-
-# Tests build when depfiles contain generated (declared) inputs.
-function test_depfile_generated_inputs() {
- setup_basic_ninja_build
-
- cat > test/build.ninja <<'EOF'
-rule filecopy
- command = cat ${in} > ${out}
-rule cattool
- depfile = out/test/depfile.d
- command = ${in} ${out}
-build out/test/generated: filecopy test/two
-build out/test/output.txt: cattool test/cattool.sh out/test/generated
-EOF
-
- cat > test/cattool.sh <<'EOF'
-OUTPUT=${!#}
-DEPFILE="$(dirname $OUTPUT)/depfile.d"
-
-cat test/one out/test/generated > $OUTPUT
-
-echo "$1 : test/one out/test/generated" > $DEPFILE
-EOF
-
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
-
- cat bazel-workspace/out/test/output.txt &> $TEST_log
- expect_log "ab"
-
- printf "z" > test/two
-
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
-
- cat bazel-workspace/out/test/output.txt &> $TEST_log
- expect_log "az"
-}
-
-# Tests error of build when depfiles contain generated (undeclared) inputs.
-function test_depfile_undeclared_generated_inputs() {
- setup_basic_ninja_build
-
- cat > test/build.ninja <<'EOF'
-rule filecopy
- command = cat ${in} > ${out}
-rule cattool
- depfile = out/test/depfile.d
- command = ${in} ${out}
-build out/test/generated: filecopy test/two
-build out/test/output.txt: cattool test/cattool.sh test/one
-EOF
-
- cat > test/cattool.sh <<'EOF'
-OUTPUT=${!#}
-DEPFILE="$(dirname $OUTPUT)/depfile.d"
-
-cat test/one out/test/generated > $OUTPUT
-
-echo "$1 : test/one out/test/generated" > $DEPFILE
-EOF
-
- ! bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "build should have failed"
-
- expect_log "depfile-declared dependency 'out/test/generated' is invalid"
-}
-
-
-# Tests Bazel behaves appropriately when depfiles aren't generated.
-function test_depfile_not_generated() {
- setup_basic_ninja_build
- cat > test/cattool.sh <<'EOF'
-OUTPUT=${!#}
-
-cat test/{one,two} > $OUTPUT
-EOF
-
- ! bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "build should have failed"
-
- expect_log "out/test/depfile.d (No such file or directory)"
-}
-
-function test_depfile_pruned_generated_input() {
- setup_basic_ninja_build
-
- cat > test/build.ninja <<'EOF'
-rule filecopy
- command = cat ${in} > ${out}
-rule cattool
- depfile = out/test/depfile.d
- command = ${in} ${out}
-build out/test/generated: filecopy test/two
-build out/test/output.txt: cattool test/cattool.sh test/one out/test/generated
-EOF
- cat > test/cattool.sh <<'EOF'
-OUTPUT=${!#}
-DEPFILE="$(dirname $OUTPUT)/depfile.d"
-
-cat test/one > $OUTPUT
-
-echo "$1 : test/cattool.sh test/one" > $DEPFILE
-EOF
-
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
- expect_log "INFO: 3 processes: 1 internal, 2 local"
-
- # test/two is a dependency of out/test/generated, which was an originally
- # declared input, but not according to the depfile.
-
- echo "z" > test/two
- # Verify the root action is not run, as test/two is not an input to the action.
- # Note that ideally this build would be a null build, as the root action
- # should no longer depend on the filecopy action. However, Skyframe currently
- # does not support action dependency pruning. Thus, the only savings is
- # an action cache hit on the root action.
- bazel build -s //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
- expect_log "INFO: 2 processes: 1 internal, 1 local"
-
- cat > test/cattool.sh <<'EOF'
-OUTPUT=${!#}
-DEPFILE="$(dirname $OUTPUT)/depfile.d"
-
-cat test/one out/test/generated > $OUTPUT
-
-echo "$1 : test/cattool.sh test/one out/test/generated" > $DEPFILE
-EOF
-
- # Build should re-execute, as cattool.sh has changed.
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
- expect_log "INFO: 2 processes: 1 internal, 1 local"
-
- # test/two should again be reflected as an input to the build via
- # the inclusion of out/test/generated in the depfile.
- echo "x" > test/two
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "build should have failed"
- expect_log "INFO: 3 processes: 1 internal, 2 local"
-}
-
-function test_external_source_dependency() {
- setup_basic_ninja_build
-
- cat > BUILD <<'EOF'
-ninja_graph(
- name = "rootgraph",
- main = "build.ninja",
- output_root = "out",
-)
-
-ninja_build(
- name = "rootbuild",
- ninja_graph = "rootgraph",
- output_groups = {
- "out" : ["out/test/output.txt"],
- }
-)
-EOF
-
- cat > build.ninja <<'EOF'
-rule cattool
- depfile = out/test/depfile.d
- command = ${in} ${out}
-build out/test/output.txt: cattool test/cattool.sh test/one external/two
-EOF
-
- cat > test/cattool.sh <<'EOF'
-OUTPUT=${!#}
-DEPFILE="$(dirname $OUTPUT)/depfile.d"
-
-cat test/one external/two > $OUTPUT
-
-echo "$1 : test/one external/two" > $DEPFILE
-EOF
- mkdir -p external
- echo "b" > external/two
-
- bazel build //:rootbuild --experimental_sibling_repository_layout --experimental_disable_external_package --experimental_ninja_actions &> $TEST_log \
- || fail "build should have succeeded"
- cat bazel-workspace/out/test/output.txt > $TEST_log
- expect_log "ab"
-
- echo "z" > external/two
-
- bazel build //:rootbuild --experimental_sibling_repository_layout --experimental_disable_external_package --experimental_ninja_actions &> $TEST_log \
- || fail "build should have succeeded"
- cat bazel-workspace/out/test/output.txt > $TEST_log
- expect_log "az"
-}
-
-# Tests a dependency on external package sources in the scenario where the
-# ninja_build target has a dependency on //external:cc_toolchain. This is a
-# regression test for a bug in which this dependency interfered with external
-# source resolution.
-function test_external_dependency_cctoolchain() {
- setup_basic_ninja_build
-
- cat > BUILD <<'EOF'
-ninja_graph(
- name = "rootgraph",
- main = "build.ninja",
- output_root = "out",
-)
-
-genrule(
- name = "g",
- cmd = "echo 'c' > $@",
- outs = ["generated.txt"],
- # This ensures rootbuild depends on cc_toolchain transitively.
- srcs = ["//external:cc_toolchain"],
-)
-
-ninja_build(
- name = "rootbuild",
- ninja_graph = "rootgraph",
- output_groups = {
- "out" : ["out/test/output.txt"],
- },
- deps_mapping = {
- "out/dummy" : ":g",
- },
-)
-EOF
-
- cat > build.ninja <<'EOF'
-rule cattool
- depfile = out/test/depfile.d
- command = ${in} ${out}
-build out/test/output.txt: cattool test/cattool.sh test/one
-EOF
-
- cat > test/cattool.sh <<'EOF'
-OUTPUT=${!#}
-DEPFILE="$(dirname $OUTPUT)/depfile.d"
-
-cat test/one external/foo/two > $OUTPUT
-
-echo "$1 : test/one external/foo/two" > $DEPFILE
-EOF
- mkdir -p external/foo
- touch external/foo/BUILD
- echo "b" > external/foo/two
-
- bazel build //:rootbuild --experimental_sibling_repository_layout --experimental_disable_external_package --experimental_ninja_actions &> $TEST_log \
- || fail "build should have succeeded"
- cat bazel-workspace/out/test/output.txt > $TEST_log
- expect_log "ab"
-
- echo "z" > external/foo/two
-
- bazel build //:rootbuild --experimental_sibling_repository_layout --experimental_disable_external_package --experimental_ninja_actions &> $TEST_log \
- || fail "build should have succeeded"
- cat bazel-workspace/out/test/output.txt > $TEST_log
- expect_log "az"
-}
-
-function test_basic_depfile_processing() {
- setup_basic_ninja_build
-
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
-
- cat bazel-workspace/out/test/output.txt &> $TEST_log
- expect_log "ab"
-
- printf "HELLO" > test/one
-
- bazel build //test:ninjabuild --experimental_ninja_actions &> $TEST_log \
- || fail "should have generated output successfully"
-
- cat bazel-workspace/out/test/output.txt &> $TEST_log
- expect_log "HELLOb"
-}
-
-function test_switching_directory_from_source_to_toplevel_output_directory() {
-
- mkdir -p out n
- touch WORKSPACE
-
- cat > BUILD <<'EOF'
-genrule(
- name = "top",
- srcs = [],
- outs = ["topo"],
- cmd = "echo TOP > $@",
-)
-
-ninja_graph(
- name = "g",
- main = "n/x.ninja",
- ninja_srcs = ["n/x.ninja"],
- output_root = "out",
- output_root_inputs = ["input"],
-)
-
-ninja_build(
- name = "b",
- ninja_graph = ":g",
- output_groups = {"o": ["out/outfile"]},
-)
-EOF
-
- cat > n/x.ninja <<'EOF'
-rule cmd
- command = ${cmd}
-
-build out/outfile: cmd out/input
- cmd = echo OUTFILE > out/outfile
-EOF
-
- echo 123 > out/input
-
- # Clean+expunge to ensure that 'out' is created as a symlink in the execroot
- bazel clean --expunge
-
- bazel build --experimental_ninja_actions //:top
- # test that the 'out' directory has been symlinked, since SymlinkForest
- # has a mode where it symlinks only the packages that have been loaded.
- execroot="$(bazel info --experimental_ninja_actions execution_root)"
- test -L "$execroot/out" || fail "$execroot/out should be a symlink"
-
- # switch the "out" directory from a source directory to a toplevel output directory
- cat > WORKSPACE <<'EOF'
-toplevel_output_directories(paths=["out"])
-EOF
-
- bazel build --experimental_ninja_actions //:b
- # when switching 'out' to an output directory, it should be a real directory
- # and not a symlink, with a symlink to the output_root_symlink in it as
- # created by the SymlinkActions in NinjaGraph.createSymlinkActions().
- execroot="$(bazel info --experimental_ninja_actions execution_root)"
- if [[ -L "$execroot/out" ]]; then fail "$execroot/out should not be a symlink"; fi
- test -d "$execroot/out" || fail "$execroot/out should be a directory"
- test -L "$execroot/out/input" || fail "$execroot/out/input should be a symlink"
-
- # now switch "out" back to a source directory, do a build, and $execroot/out
- # should be back to a symlink to the directory in the source tree.
- echo "" > WORKSPACE
- bazel build --experimental_ninja_actions //:top
- execroot="$(bazel info --experimental_ninja_actions execution_root)"
- test -L "$execroot/out" || fail "$execroot/out should be a symlink"
-}
-
-run_suite "ninja_build rule tests"
