Add the support to LLVM's Context Sensitive FDO
LLVM recently implemented Context Sensitive FDO/PGO (CSFDO/CSPGO) which uses a
post-inline instrumentation to record more accurate profile information for the
inlined code. It improves the effectiveness of post-inline optimizations.
This CL adds the support to CSFDO:
(1) Add new options:
--cs_fdo_instrument=<csfdo_instru_path>,
--cs_profile=<csfdo_label>
(2) Add two features: CS_FDO_INSTRUMENT and CS_FDO_OPTIMIZE.
(2) Add a new BranchFdoMode: LLVM_CS_FDO.
(3) Merge CSFDO profile and regular FDO profile.
(4) Pass CSFDO options to the back-end invocations.
(5) Distribute the profile to build hosts in LLVM_CS_FDO optimize build. This is
not needed for regular FDO as the profile is read in during pre-LTO passes. We
need this step in CSFDO because the profile is read-in and used in the back-end.
RELNOTES: Add new options --cs_fdo_instrument and --cs_profile to support
LLVM's context-sensitive FDO (CSFDO).
PiperOrigin-RevId: 242685802
diff --git a/site/docs/cc-toolchain-config-reference.md b/site/docs/cc-toolchain-config-reference.md
index 5e8b05e..bef1804 100644
--- a/site/docs/cc-toolchain-config-reference.md
+++ b/site/docs/cc-toolchain-config-reference.md
@@ -976,6 +976,35 @@
of <code>.o</code> files and the compiler and linker need to know this.
</td>
</tr>
+ <tr>
+ <td><strong><code>fdo_instrument_path</code></strong>
+ </td>
+ <td>compile, link</td>
+ <td> Path to the directory that stores FDO instrumentation profile.
+ </td>
+ </tr>
+ <tr>
+ <td><strong><code>fdo_profile_path</code></strong>
+ </td>
+ <td>compile</td>
+ <td> Path to FDO profile.
+ </td>
+ </tr>
+ <tr>
+ <td><strong><code>fdo_prefetch_hints_path</code></strong>
+ </td>
+ <td>compile</td>
+ <td> Path to the cache prefetch profile.
+ </td>
+ </tr>
+ <tr>
+ <td><strong><code>csfdo_instrument_path</code></strong>
+ </td>
+ <td>compile, link</td>
+ <td> Path to the directory that stores context sensitive FDO
+ instrumentation profile.
+ </td>
+ </tr>
</table>
@@ -1094,6 +1123,8 @@
<li>Adds <code>include_paths</code> (if not present) feature to the top of the toolchain</li>
<li>Adds <code>fdo_instrument</code> (if not present) feature to the top of the toolchain</li>
<li>Adds <code>fdo_optimize</code> (if not present) feature to the top of the toolchain</li>
+ <li>Adds <code>cs_fdo_instrument</code> (if not present) feature to the top of the toolchain</li>
+ <li>Adds <code>cs_fdo_optimize</code> (if not present) feature to the top of the toolchain</li>
<li>Adds <code>fdo_prefetch_hints</code> (if not present) feature to the top of the toolchain</li>
<li>Adds <code>autofdo</code> (if not present) feature to the top of the toolchain</li>
<li>Adds <code>build_interface_libraries</code> (if not present) feature to the top of the toolchain</li>
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java
index 447539b..a19f37e 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java
@@ -844,14 +844,19 @@
allFeatures.addAll(getCoverageFeatures(cppConfiguration));
- String fdoInstrument = cppConfiguration.getFdoInstrument();
- if (fdoInstrument != null && !allUnsupportedFeatures.contains(CppRuleClasses.FDO_INSTRUMENT)) {
- allFeatures.add(CppRuleClasses.FDO_INSTRUMENT);
+ if (!allUnsupportedFeatures.contains(CppRuleClasses.FDO_INSTRUMENT)) {
+ if (cppConfiguration.getFdoInstrument() != null) {
+ allFeatures.add(CppRuleClasses.FDO_INSTRUMENT);
+ } else {
+ if (cppConfiguration.getCSFdoInstrument() != null) {
+ allFeatures.add(CppRuleClasses.CS_FDO_INSTRUMENT);
+ }
+ }
}
FdoContext.BranchFdoProfile branchFdoProvider = toolchain.getFdoContext().getBranchFdoProfile();
if (branchFdoProvider != null && cppConfiguration.getCompilationMode() == CompilationMode.OPT) {
- if (branchFdoProvider.isLlvmFdo()
+ if ((branchFdoProvider.isLlvmFdo() || branchFdoProvider.isLlvmCSFdo())
&& !allUnsupportedFeatures.contains(CppRuleClasses.FDO_OPTIMIZE)) {
allFeatures.add(CppRuleClasses.FDO_OPTIMIZE);
// For LLVM, support implicit enabling of ThinLTO for FDO unless it has been
@@ -861,6 +866,9 @@
allFeatures.add(CppRuleClasses.ENABLE_FDO_THINLTO);
}
}
+ if (branchFdoProvider.isLlvmCSFdo()) {
+ allFeatures.add(CppRuleClasses.CS_FDO_OPTIMIZE);
+ }
if (branchFdoProvider.isAutoFdo()) {
allFeatures.add(CppRuleClasses.AUTOFDO);
// For LLVM, support implicit enabling of ThinLTO for AFDO unless it has been
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java
index e1ddaf1..81e7d5c 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java
@@ -94,11 +94,16 @@
FeatureConfiguration featureConfiguration,
FdoContext fdoContext,
String fdoInstrument,
+ String csFdoInstrument,
CppConfiguration cppConfiguration) {
if (featureConfiguration.isEnabled(CppRuleClasses.FDO_INSTRUMENT)) {
variablesBuilder.put(
CompileBuildVariables.FDO_INSTRUMENT_PATH.getVariableName(), fdoInstrument);
}
+ if (featureConfiguration.isEnabled(CppRuleClasses.CS_FDO_INSTRUMENT)) {
+ variablesBuilder.put(
+ CompileBuildVariables.CS_FDO_INSTRUMENT_PATH.getVariableName(), csFdoInstrument);
+ }
// FDO is disabled -> do nothing.
Preconditions.checkNotNull(fdoContext);
@@ -126,7 +131,7 @@
branchFdoProfile.getProfileArtifact().getExecPathString());
}
if (featureConfiguration.isEnabled(CppRuleClasses.FDO_OPTIMIZE)) {
- if (branchFdoProfile.isLlvmFdo()) {
+ if (branchFdoProfile.isLlvmFdo() || branchFdoProfile.isLlvmCSFdo()) {
variablesBuilder.put(
CompileBuildVariables.FDO_PROFILE_PATH.getVariableName(),
branchFdoProfile.getProfileArtifact().getExecPathString());
@@ -1440,6 +1445,7 @@
featureConfiguration,
fdoContext,
cppConfiguration.getFdoInstrument(),
+ cppConfiguration.getCSFdoInstrument(),
cppConfiguration);
}
return CompileBuildVariables.setupVariablesOrReportRuleError(
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainAttributesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainAttributesProvider.java
index 6f0a7b6..ff03413 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainAttributesProvider.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainAttributesProvider.java
@@ -78,7 +78,6 @@
private final String cpu;
private final Artifact ifsoBuilder;
private final Artifact linkDynamicLibraryTool;
- private final FdoProfileProvider fdoOptimizeProvider;
private final TransitiveInfoCollection fdoOptimize;
private final ImmutableList<Artifact> fdoOptimizeArtifacts;
private final FdoPrefetchHintsProvider fdoPrefetch;
@@ -92,7 +91,9 @@
private final AdditionalBuildVariablesComputer additionalBuildVariablesComputer;
private final CcToolchainConfigInfo ccToolchainConfigInfo;
private final String toolchainIdentifier;
+ private final FdoProfileProvider fdoOptimizeProvider;
private final FdoProfileProvider fdoProfileProvider;
+ private final FdoProfileProvider csFdoProfileProvider;
private final FdoProfileProvider xfdoProfileProvider;
private final Label ccToolchainLabel;
private final TransitiveInfoCollection staticRuntimeLib;
@@ -167,6 +168,9 @@
this.fdoProfileProvider =
ruleContext.getPrerequisite(
CcToolchainRule.FDO_PROFILE_ATTR, Mode.TARGET, FdoProfileProvider.PROVIDER);
+ this.csFdoProfileProvider =
+ ruleContext.getPrerequisite(
+ CcToolchainRule.CSFDO_PROFILE_ATTR, Mode.TARGET, FdoProfileProvider.PROVIDER);
this.xfdoProfileProvider =
ruleContext.getPrerequisite(
CcToolchainRule.XFDO_PROFILE_ATTR, Mode.TARGET, FdoProfileProvider.PROVIDER);
@@ -348,6 +352,10 @@
return fdoProfileProvider;
}
+ public FdoProfileProvider getCSFdoProfileProvider() {
+ return csFdoProfileProvider;
+ }
+
public FdoProfileProvider getXFdoProfileProvider() {
return xfdoProfileProvider;
}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java
index f25ee30..b845343 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java
@@ -613,6 +613,11 @@
return cppConfiguration;
}
+ /** Return context-sensitive fdo instrumentation path. */
+ public String getCSFdoInstrument() {
+ return cppConfiguration.getCSFdoInstrument();
+ }
+
/** Returns build variables to be templated into the crosstool. */
public CcToolchainVariables getBuildVariables(
BuildOptions buildOptions, CppConfiguration cppConfiguration) {
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java
index 4a4949a..ff9ccc4 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java
@@ -42,6 +42,7 @@
public static final String TARGET_LIBC_TOP_ATTR = ":target_libc_top";
public static final String FDO_OPTIMIZE_ATTR = ":fdo_optimize";
public static final String FDO_PROFILE_ATTR = ":fdo_profile";
+ public static final String CSFDO_PROFILE_ATTR = ":csfdo_profile";
public static final String XFDO_PROFILE_ATTR = ":xfdo_profile";
public static final String TOOLCHAIN_CONFIG_ATTR = "toolchain_config";
@@ -101,6 +102,12 @@
(rule, attributes, cppConfig) ->
cppConfig.getFdoProfileLabelUnsafeSinceItCanReturnValueFromWrongConfiguration());
+ private static final LabelLateBoundDefault<?> CSFDO_PROFILE_VALUE =
+ LabelLateBoundDefault.fromTargetConfiguration(
+ CppConfiguration.class,
+ null,
+ (rule, attributes, cppConfig) -> cppConfig.getCSFdoProfileLabel());
+
private static final LabelLateBoundDefault<?> XFDO_PROFILE_VALUE =
LabelLateBoundDefault.fromTargetConfiguration(
CppConfiguration.class,
@@ -324,6 +331,11 @@
.mandatoryProviders(ImmutableList.of(FdoProfileProvider.PROVIDER.id()))
.value(FDO_PROFILE_VALUE))
.add(
+ attr(CSFDO_PROFILE_ATTR, LABEL)
+ .allowedRuleClasses("fdo_profile")
+ .mandatoryProviders(ImmutableList.of(FdoProfileProvider.PROVIDER.id()))
+ .value(CSFDO_PROFILE_VALUE))
+ .add(
attr(":fdo_prefetch_hints", LABEL)
.allowedRuleClasses("fdo_prefetch_hints")
.mandatoryProviders(ImmutableList.of(FdoPrefetchHintsProvider.PROVIDER.id()))
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CompileBuildVariables.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CompileBuildVariables.java
index e8d823a..3eb49bb 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CompileBuildVariables.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CompileBuildVariables.java
@@ -91,6 +91,8 @@
FDO_INSTRUMENT_PATH("fdo_instrument_path"),
/** Path to the fdo profile artifact */
FDO_PROFILE_PATH("fdo_profile_path"),
+ /** Path to the context sensitive fdo instrument artifact */
+ CS_FDO_INSTRUMENT_PATH("cs_fdo_instrument_path"),
/** Path to the cache prefetch profile artifact */
FDO_PREFETCH_HINTS_PATH("fdo_prefetch_hints_path"),
/** Variable for includes that compiler needs to include into sources. */
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java
index e35bd74..2e0c3d6 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java
@@ -297,6 +297,47 @@
" }",
" }")));
}
+
+ if (!existingFeatureNames.contains(CppRuleClasses.CS_FDO_INSTRUMENT)) {
+ featureBuilder.add(
+ getFeature(
+ Joiner.on("\n")
+ .join(
+ " name: 'cs_fdo_instrument'",
+ " provides: 'csprofile'",
+ " flag_set {",
+ " action: 'c-compile'",
+ " action: 'c++-compile'",
+ " action: 'lto-backend'",
+ " action: 'c++-link-dynamic-library'",
+ " action: 'c++-link-nodeps-dynamic-library'",
+ " action: 'c++-link-executable'",
+ " flag_group {",
+ " expand_if_all_available: 'cs_fdo_instrument_path'",
+ " flag: '-fcs-profile-generate=%{cs_fdo_instrument_path}'",
+ " }",
+ " }")));
+ }
+
+ if (!existingFeatureNames.contains(CppRuleClasses.CS_FDO_OPTIMIZE)) {
+ featureBuilder.add(
+ getFeature(
+ Joiner.on("\n")
+ .join(
+ " name: 'cs_fdo_optimize'",
+ " provides: 'csprofile'",
+ " flag_set {",
+ " action: 'lto-backend'",
+ " flag_group {",
+ " expand_if_all_available: 'fdo_profile_path'",
+ " flag: '-fprofile-use=%{fdo_profile_path}'",
+ " flag: '-Xclang-only=-Wno-profile-instr-unprofiled'",
+ " flag: '-Xclang-only=-Wno-profile-instr-out-of-date'",
+ " flag: '-fprofile-correction'",
+ " }",
+ " }")));
+ }
+
if (!existingFeatureNames.contains(CppRuleClasses.FDO_PREFETCH_HINTS)) {
featureBuilder.add(
getFeature(
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java
index c797ca1..afddfa4 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java
@@ -302,6 +302,10 @@
return cppOptions.isFdo();
}
+ public boolean isCSFdo() {
+ return cppOptions.isCSFdo();
+ }
+
/**
* Returns whether or not to strip the binaries.
*/
@@ -540,6 +544,10 @@
return fdoOptimizeLabel;
}
+ public String getCSFdoInstrument() {
+ return cppOptions.csFdoInstrumentForBuild;
+ }
+
Label getFdoPrefetchHintsLabel() {
if (isThisHostConfigurationDoNotUseWillBeRemovedFor129045294()) {
// We don't want FDO in the host configuration
@@ -566,6 +574,10 @@
return cppOptions.fdoProfileLabel;
}
+ public Label getCSFdoProfileLabel() {
+ return cppOptions.csFdoProfileLabel;
+ }
+
/**
* @deprecated Unsafe because it returns a value from target configuration even in the host
* configuration.
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java
index c9634bf..61b10ea 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java
@@ -599,6 +599,9 @@
return featureConfiguration.isEnabled(CppRuleClasses.XBINARYFDO) ? "XFDO" : null;
}
}
+ if (cppConfiguration.isCSFdo()) {
+ return "CSFDO";
+ }
if (cppConfiguration.isFdo()) {
return "FDO";
}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java
index 9322ef4..af39bbf 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java
@@ -420,6 +420,18 @@
}
@Option(
+ name = "cs_fdo_instrument",
+ defaultValue = "null",
+ implicitRequirements = {"--copt=-Wno-error"},
+ documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ help =
+ "Generate binaries with context sensitive FDO instrumentation. With Clang/LLVM compiler, "
+ + "it also accepts the directory name under which the raw profile file(s) will be "
+ + "dumped at runtime.")
+ public String csFdoInstrumentForBuild;
+
+ @Option(
name = "xbinary_fdo",
defaultValue = "null",
documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS,
@@ -462,6 +474,18 @@
public Label fdoProfileLabel;
@Option(
+ name = "cs_fdo_profile",
+ defaultValue = "null",
+ category = "flags",
+ converter = LabelConverter.class,
+ documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ help =
+ "The cs_fdo_profile representing the context sensitive profile to be used for"
+ + " optimization.")
+ public Label csFdoProfileLabel;
+
+ @Option(
name = "enable_fdo_profile_absolute_path",
defaultValue = "true",
documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS,
@@ -905,6 +929,7 @@
host.stripBinaries = StripMode.ALWAYS;
host.fdoOptimizeForBuild = fdoOptimizeForBuild;
host.fdoProfileLabel = fdoProfileLabel;
+ host.csFdoProfileLabel = csFdoProfileLabel;
host.xfdoProfileLabel = xfdoProfileLabel;
host.inmemoryDotdFiles = inmemoryDotdFiles;
@@ -928,4 +953,10 @@
public boolean isFdo() {
return getFdoOptimize() != null || fdoInstrumentForBuild != null || fdoProfileLabel != null;
}
+
+ /** Returns true if targets under this configuration should apply CSFdo. */
+ public boolean isCSFdo() {
+ return ((getFdoOptimize() != null || fdoProfileLabel != null)
+ && (csFdoInstrumentForBuild != null || csFdoProfileLabel != null));
+ }
}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java
index 6686bff..87c280a 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java
@@ -348,9 +348,15 @@
*/
public static final String FDO_INSTRUMENT = "fdo_instrument";
+ /** A string constant for the cs_fdo_instrument feature. */
+ public static final String CS_FDO_INSTRUMENT = "cs_fdo_instrument";
+
/** A string constant for the fdo_optimize feature. */
public static final String FDO_OPTIMIZE = "fdo_optimize";
+ /** A string constant for the cs_fdo_optimize feature. */
+ public static final String CS_FDO_OPTIMIZE = "cs_fdo_optimize";
+
/** A string constant for the cache prefetch hints feature. */
public static final String FDO_PREFETCH_HINTS = "fdo_prefetch_hints";
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoContext.java
index 7fed311..2b7766a 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoContext.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoContext.java
@@ -40,6 +40,9 @@
/** Instrumentation-based FDO implemented on LLVM. */
LLVM_FDO,
+
+ /** Instrumentation-based Context Sensitive FDO implemented on LLVM. */
+ LLVM_CS_FDO,
}
/** A POJO encapsulating the branch profiling configuration. */
@@ -68,6 +71,10 @@
return branchFdoMode == BranchFdoMode.LLVM_FDO;
}
+ public boolean isLlvmCSFdo() {
+ return branchFdoMode == BranchFdoMode.LLVM_CS_FDO;
+ }
+
public Artifact getProfileArtifact() {
return profileArtifact;
}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoHelper.java
index 124ac70..687373a 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoHelper.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoHelper.java
@@ -44,6 +44,7 @@
CppToolchainInfo toolchainInfo)
throws InterruptedException, RuleErrorException {
FdoInputFile fdoInputFile = null;
+ FdoInputFile csFdoInputFile = null;
FdoInputFile prefetchHints = null;
Artifact protoProfileArtifact = null;
Pair<FdoInputFile, Artifact> fdoInputs = null;
@@ -87,6 +88,23 @@
!= null) {
fdoInputs = getFdoInputs(ruleContext, attributes.getXFdoProfileProvider());
}
+
+ Pair<FdoInputFile, Artifact> csFdoInputs = null;
+ if (cppConfiguration.getCSFdoProfileLabel() != null) {
+ csFdoInputs = getFdoInputs(ruleContext, attributes.getCSFdoProfileProvider());
+ }
+ if (csFdoInputs != null) {
+ csFdoInputFile = csFdoInputs.getFirst();
+ }
+ }
+
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+
+ if (fdoInputs != null) {
+ fdoInputFile = fdoInputs.getFirst();
+ protoProfileArtifact = fdoInputs.getSecond();
}
if (ruleContext.hasErrors()) {
@@ -115,6 +133,12 @@
ruleContext.ruleError("invalid extension for FDO profile file.");
return null;
}
+ // Check if this is LLVM_CS_FDO
+ if (branchFdoMode == BranchFdoMode.LLVM_FDO) {
+ if (csFdoInputFile != null) {
+ branchFdoMode = BranchFdoMode.LLVM_CS_FDO;
+ }
+ }
if (branchFdoMode != BranchFdoMode.XBINARY_FDO
&& cppConfiguration.getXFdoProfileLabelUnsafeSinceItCanReturnValueFromWrongConfiguration()
!= null) {
@@ -130,7 +154,7 @@
if (branchFdoMode == BranchFdoMode.LLVM_FDO) {
profileArtifact =
convertLLVMRawProfileToIndexed(
- attributes, fdoInputFile, toolchainInfo, ruleContext, cppConfiguration);
+ attributes, fdoInputFile, toolchainInfo, ruleContext, cppConfiguration, "fdo");
if (ruleContext.hasErrors()) {
return null;
}
@@ -144,6 +168,33 @@
profileArtifact,
fdoInputFile,
"Symlinking FDO profile " + fdoInputFile.getBasename());
+ } else if (branchFdoMode == BranchFdoMode.LLVM_CS_FDO) {
+ Artifact nonCSProfileArtifact =
+ convertLLVMRawProfileToIndexed(
+ attributes, fdoInputFile, toolchainInfo, ruleContext, cppConfiguration, "fdo");
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+ Artifact csProfileArtifact =
+ convertLLVMRawProfileToIndexed(
+ attributes, csFdoInputFile, toolchainInfo, ruleContext, cppConfiguration, "csfdo");
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+ if (nonCSProfileArtifact != null && csProfileArtifact != null) {
+ profileArtifact =
+ mergeLLVMProfiles(
+ attributes,
+ toolchainInfo,
+ ruleContext,
+ nonCSProfileArtifact,
+ csProfileArtifact,
+ "mergedfdo",
+ "MergedCS.profdata");
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+ }
}
branchFdoProfile =
new FdoContext.BranchFdoProfile(branchFdoMode, profileArtifact, protoProfileArtifact);
@@ -209,6 +260,43 @@
}
}
+ /** This function merges profile1 and profile2 and generates mergedOutput. */
+ private static Artifact mergeLLVMProfiles(
+ CcToolchainAttributesProvider attributes,
+ CppToolchainInfo toolchainInfo,
+ RuleContext ruleContext,
+ Artifact profile1,
+ Artifact profile2,
+ String fdoUniqueArtifactName,
+ String mergedOutput) {
+ Artifact profileArtifact =
+ ruleContext.getUniqueDirectoryArtifact(
+ fdoUniqueArtifactName, mergedOutput, ruleContext.getBinOrGenfilesDirectory());
+
+ // Merge LLVM profiles.
+ ruleContext.registerAction(
+ new SpawnAction.Builder()
+ .addInput(profile1)
+ .addInput(profile2)
+ .addTransitiveInputs(attributes.getAllFilesMiddleman())
+ .addOutput(profileArtifact)
+ .useDefaultShellEnvironment()
+ .setExecutable(toolchainInfo.getToolPathFragment(Tool.LLVM_PROFDATA))
+ .setProgressMessage("LLVMProfDataAction: Generating %s", profileArtifact.prettyPrint())
+ .setMnemonic("LLVMProfDataMergeAction")
+ .addCommandLine(
+ CustomCommandLine.builder()
+ .add("merge")
+ .add("-o")
+ .addExecPath(profileArtifact)
+ .addExecPath(profile1)
+ .addExecPath(profile2)
+ .build())
+ .build(ruleContext));
+
+ return profileArtifact;
+ }
+
/*
* This function checks the format of the input profile data and converts it to
* the indexed format (.profdata) if necessary.
@@ -218,14 +306,15 @@
FdoInputFile fdoProfile,
CppToolchainInfo toolchainInfo,
RuleContext ruleContext,
- CppConfiguration cppConfiguration) {
+ CppConfiguration cppConfiguration,
+ String fdoUniqueArtifactName) {
if (cppConfiguration.isThisHostConfigurationDoNotUseWillBeRemovedFor129045294()) {
return null;
}
Artifact profileArtifact =
ruleContext.getUniqueDirectoryArtifact(
- "fdo",
+ fdoUniqueArtifactName,
getLLVMProfileFileName(fdoProfile, CppFileTypes.LLVM_PROFILE),
ruleContext.getBinOrGenfilesDirectory());
@@ -263,12 +352,14 @@
}
rawProfileArtifact =
ruleContext.getUniqueDirectoryArtifact(
- "fdo", rawProfileFileName, ruleContext.getBinOrGenfilesDirectory());
+ fdoUniqueArtifactName, rawProfileFileName, ruleContext.getBinOrGenfilesDirectory());
// Symlink to the zipped profile file to extract the contents.
Artifact zipProfileArtifact =
ruleContext.getUniqueDirectoryArtifact(
- "fdo", fdoProfile.getBasename(), ruleContext.getBinOrGenfilesDirectory());
+ fdoUniqueArtifactName,
+ fdoProfile.getBasename(),
+ ruleContext.getBinOrGenfilesDirectory());
symlinkTo(
ruleContext,
zipProfileArtifact,
@@ -297,7 +388,7 @@
} else {
rawProfileArtifact =
ruleContext.getUniqueDirectoryArtifact(
- "fdo",
+ fdoUniqueArtifactName,
getLLVMProfileFileName(fdoProfile, CppFileTypes.LLVM_PROFILE_RAW),
ruleContext.getBinOrGenfilesDirectory());
symlinkTo(
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkBuildVariables.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkBuildVariables.java
index ac2da3d..a79102e 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkBuildVariables.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkBuildVariables.java
@@ -75,7 +75,11 @@
* Presence of this variable indicates that files were compiled with fission (debug info is in
* .dwo files instead of .o files and linker needs to know).
*/
- IS_USING_FISSION("is_using_fission");
+ IS_USING_FISSION("is_using_fission"),
+ /** Path to the fdo instrument. */
+ FDO_INSTRUMENT_PATH("fdo_instrument_path"),
+ /** Path to the context sensitive fdo instrument. */
+ CS_FDO_INSTRUMENT_PATH("cs_fdo_instrument_path");
private final String variableName;
@@ -224,7 +228,11 @@
Preconditions.checkArgument(fdoContext.getBranchFdoProfile() == null);
String fdoInstrument = cppConfiguration.getFdoInstrument();
Preconditions.checkNotNull(fdoInstrument);
- buildVariables.addStringVariable("fdo_instrument_path", fdoInstrument);
+ buildVariables.addStringVariable(FDO_INSTRUMENT_PATH.getVariableName(), fdoInstrument);
+ } else if (featureConfiguration.isEnabled(CppRuleClasses.CS_FDO_INSTRUMENT)) {
+ String csFdoInstrument = ccToolchainProvider.getCSFdoInstrument();
+ Preconditions.checkNotNull(csFdoInstrument);
+ buildVariables.addStringVariable(CS_FDO_INSTRUMENT_PATH.getVariableName(), csFdoInstrument);
}
Iterable<String> userLinkFlagsWithLtoIndexingIfNeeded;
diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendArtifacts.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendArtifacts.java
index ec921b2..c8b2121 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendArtifacts.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendArtifacts.java
@@ -253,6 +253,12 @@
buildVariablesBuilder.addStringVariable(
"thinlto_input_bitcode_file", bitcodeFile.getExecPath().toString());
addProfileForLtoBackend(builder, fdoContext, featureConfiguration, buildVariablesBuilder);
+ // Add the context sensitive instrument path to the backend.
+ if (featureConfiguration.isEnabled(CppRuleClasses.CS_FDO_INSTRUMENT)) {
+ buildVariablesBuilder.addStringVariable(
+ CompileBuildVariables.CS_FDO_INSTRUMENT_PATH.getVariableName(),
+ ccToolchain.getCSFdoInstrument());
+ }
if (generateDwo) {
dwoFile =
@@ -303,6 +309,7 @@
builder.addInput(fdoContext.getPrefetchHintsArtifact());
}
if (!featureConfiguration.isEnabled(CppRuleClasses.AUTOFDO)
+ && !featureConfiguration.isEnabled(CppRuleClasses.CS_FDO_OPTIMIZE)
&& !featureConfiguration.isEnabled(CppRuleClasses.XBINARYFDO)) {
return;
}
