| //===- ProfileSummaryInfo.cpp - Global profile summary information --------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file contains a pass that provides access to the global profile summary |
| // information. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "llvm/Analysis/ProfileSummaryInfo.h" |
| #include "llvm/Analysis/BlockFrequencyInfo.h" |
| #include "llvm/IR/BasicBlock.h" |
| #include "llvm/IR/Instructions.h" |
| #include "llvm/IR/Metadata.h" |
| #include "llvm/IR/Module.h" |
| #include "llvm/IR/ProfileSummary.h" |
| #include "llvm/InitializePasses.h" |
| #include "llvm/ProfileData/ProfileCommon.h" |
| #include "llvm/Support/CommandLine.h" |
| using namespace llvm; |
| |
| // The following two parameters determine the threshold for a count to be |
| // considered hot/cold. These two parameters are percentile values (multiplied |
| // by 10000). If the counts are sorted in descending order, the minimum count to |
| // reach ProfileSummaryCutoffHot gives the threshold to determine a hot count. |
| // Similarly, the minimum count to reach ProfileSummaryCutoffCold gives the |
| // threshold for determining cold count (everything <= this threshold is |
| // considered cold). |
| |
| static cl::opt<int> ProfileSummaryCutoffHot( |
| "profile-summary-cutoff-hot", cl::Hidden, cl::init(990000), cl::ZeroOrMore, |
| cl::desc("A count is hot if it exceeds the minimum count to" |
| " reach this percentile of total counts.")); |
| |
| static cl::opt<int> ProfileSummaryCutoffCold( |
| "profile-summary-cutoff-cold", cl::Hidden, cl::init(999999), cl::ZeroOrMore, |
| cl::desc("A count is cold if it is below the minimum count" |
| " to reach this percentile of total counts.")); |
| |
| static cl::opt<unsigned> ProfileSummaryHugeWorkingSetSizeThreshold( |
| "profile-summary-huge-working-set-size-threshold", cl::Hidden, |
| cl::init(15000), cl::ZeroOrMore, |
| cl::desc("The code working set size is considered huge if the number of" |
| " blocks required to reach the -profile-summary-cutoff-hot" |
| " percentile exceeds this count.")); |
| |
| static cl::opt<unsigned> ProfileSummaryLargeWorkingSetSizeThreshold( |
| "profile-summary-large-working-set-size-threshold", cl::Hidden, |
| cl::init(12500), cl::ZeroOrMore, |
| cl::desc("The code working set size is considered large if the number of" |
| " blocks required to reach the -profile-summary-cutoff-hot" |
| " percentile exceeds this count.")); |
| |
| // The next two options override the counts derived from summary computation and |
| // are useful for debugging purposes. |
| static cl::opt<int> ProfileSummaryHotCount( |
| "profile-summary-hot-count", cl::ReallyHidden, cl::ZeroOrMore, |
| cl::desc("A fixed hot count that overrides the count derived from" |
| " profile-summary-cutoff-hot")); |
| |
| static cl::opt<int> ProfileSummaryColdCount( |
| "profile-summary-cold-count", cl::ReallyHidden, cl::ZeroOrMore, |
| cl::desc("A fixed cold count that overrides the count derived from" |
| " profile-summary-cutoff-cold")); |
| |
| static cl::opt<bool> PartialProfile( |
| "partial-profile", cl::Hidden, cl::init(false), |
| cl::desc("Specify the current profile is used as a partial profile.")); |
| |
| cl::opt<bool> ScalePartialSampleProfileWorkingSetSize( |
| "scale-partial-sample-profile-working-set-size", cl::Hidden, cl::init(true), |
| cl::desc( |
| "If true, scale the working set size of the partial sample profile " |
| "by the partial profile ratio to reflect the size of the program " |
| "being compiled.")); |
| |
| static cl::opt<double> PartialSampleProfileWorkingSetSizeScaleFactor( |
| "partial-sample-profile-working-set-size-scale-factor", cl::Hidden, |
| cl::init(0.008), |
| cl::desc("The scale factor used to scale the working set size of the " |
| "partial sample profile along with the partial profile ratio. " |
| "This includes the factor of the profile counter per block " |
| "and the factor to scale the working set size to use the same " |
| "shared thresholds as PGO.")); |
| |
| // The profile summary metadata may be attached either by the frontend or by |
| // any backend passes (IR level instrumentation, for example). This method |
| // checks if the Summary is null and if so checks if the summary metadata is now |
| // available in the module and parses it to get the Summary object. |
| void ProfileSummaryInfo::refresh() { |
| if (hasProfileSummary()) |
| return; |
| // First try to get context sensitive ProfileSummary. |
| auto *SummaryMD = M->getProfileSummary(/* IsCS */ true); |
| if (SummaryMD) |
| Summary.reset(ProfileSummary::getFromMD(SummaryMD)); |
| |
| if (!hasProfileSummary()) { |
| // This will actually return PSK_Instr or PSK_Sample summary. |
| SummaryMD = M->getProfileSummary(/* IsCS */ false); |
| if (SummaryMD) |
| Summary.reset(ProfileSummary::getFromMD(SummaryMD)); |
| } |
| if (!hasProfileSummary()) |
| return; |
| computeThresholds(); |
| } |
| |
| Optional<uint64_t> ProfileSummaryInfo::getProfileCount( |
| const CallBase &Call, BlockFrequencyInfo *BFI, bool AllowSynthetic) const { |
| assert((isa<CallInst>(Call) || isa<InvokeInst>(Call)) && |
| "We can only get profile count for call/invoke instruction."); |
| if (hasSampleProfile()) { |
| // In sample PGO mode, check if there is a profile metadata on the |
| // instruction. If it is present, determine hotness solely based on that, |
| // since the sampled entry count may not be accurate. If there is no |
| // annotated on the instruction, return None. |
| uint64_t TotalCount; |
| if (Call.extractProfTotalWeight(TotalCount)) |
| return TotalCount; |
| return None; |
| } |
| if (BFI) |
| return BFI->getBlockProfileCount(Call.getParent(), AllowSynthetic); |
| return None; |
| } |
| |
| /// Returns true if the function's entry is hot. If it returns false, it |
| /// either means it is not hot or it is unknown whether it is hot or not (for |
| /// example, no profile data is available). |
| bool ProfileSummaryInfo::isFunctionEntryHot(const Function *F) const { |
| if (!F || !hasProfileSummary()) |
| return false; |
| auto FunctionCount = F->getEntryCount(); |
| // FIXME: The heuristic used below for determining hotness is based on |
| // preliminary SPEC tuning for inliner. This will eventually be a |
| // convenience method that calls isHotCount. |
| return FunctionCount && isHotCount(FunctionCount.getCount()); |
| } |
| |
| /// Returns true if the function contains hot code. This can include a hot |
| /// function entry count, hot basic block, or (in the case of Sample PGO) |
| /// hot total call edge count. |
| /// If it returns false, it either means it is not hot or it is unknown |
| /// (for example, no profile data is available). |
| bool ProfileSummaryInfo::isFunctionHotInCallGraph( |
| const Function *F, BlockFrequencyInfo &BFI) const { |
| if (!F || !hasProfileSummary()) |
| return false; |
| if (auto FunctionCount = F->getEntryCount()) |
| if (isHotCount(FunctionCount.getCount())) |
| return true; |
| |
| if (hasSampleProfile()) { |
| uint64_t TotalCallCount = 0; |
| for (const auto &BB : *F) |
| for (const auto &I : BB) |
| if (isa<CallInst>(I) || isa<InvokeInst>(I)) |
| if (auto CallCount = getProfileCount(cast<CallBase>(I), nullptr)) |
| TotalCallCount += CallCount.getValue(); |
| if (isHotCount(TotalCallCount)) |
| return true; |
| } |
| for (const auto &BB : *F) |
| if (isHotBlock(&BB, &BFI)) |
| return true; |
| return false; |
| } |
| |
| /// Returns true if the function only contains cold code. This means that |
| /// the function entry and blocks are all cold, and (in the case of Sample PGO) |
| /// the total call edge count is cold. |
| /// If it returns false, it either means it is not cold or it is unknown |
| /// (for example, no profile data is available). |
| bool ProfileSummaryInfo::isFunctionColdInCallGraph( |
| const Function *F, BlockFrequencyInfo &BFI) const { |
| if (!F || !hasProfileSummary()) |
| return false; |
| if (auto FunctionCount = F->getEntryCount()) |
| if (!isColdCount(FunctionCount.getCount())) |
| return false; |
| |
| if (hasSampleProfile()) { |
| uint64_t TotalCallCount = 0; |
| for (const auto &BB : *F) |
| for (const auto &I : BB) |
| if (isa<CallInst>(I) || isa<InvokeInst>(I)) |
| if (auto CallCount = getProfileCount(cast<CallBase>(I), nullptr)) |
| TotalCallCount += CallCount.getValue(); |
| if (!isColdCount(TotalCallCount)) |
| return false; |
| } |
| for (const auto &BB : *F) |
| if (!isColdBlock(&BB, &BFI)) |
| return false; |
| return true; |
| } |
| |
| bool ProfileSummaryInfo::isFunctionHotnessUnknown(const Function &F) const { |
| assert(hasPartialSampleProfile() && "Expect partial sample profile"); |
| return !F.getEntryCount().hasValue(); |
| } |
| |
| template <bool isHot> |
| bool ProfileSummaryInfo::isFunctionHotOrColdInCallGraphNthPercentile( |
| int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const { |
| if (!F || !hasProfileSummary()) |
| return false; |
| if (auto FunctionCount = F->getEntryCount()) { |
| if (isHot && |
| isHotCountNthPercentile(PercentileCutoff, FunctionCount.getCount())) |
| return true; |
| if (!isHot && |
| !isColdCountNthPercentile(PercentileCutoff, FunctionCount.getCount())) |
| return false; |
| } |
| if (hasSampleProfile()) { |
| uint64_t TotalCallCount = 0; |
| for (const auto &BB : *F) |
| for (const auto &I : BB) |
| if (isa<CallInst>(I) || isa<InvokeInst>(I)) |
| if (auto CallCount = getProfileCount(cast<CallBase>(I), nullptr)) |
| TotalCallCount += CallCount.getValue(); |
| if (isHot && isHotCountNthPercentile(PercentileCutoff, TotalCallCount)) |
| return true; |
| if (!isHot && !isColdCountNthPercentile(PercentileCutoff, TotalCallCount)) |
| return false; |
| } |
| for (const auto &BB : *F) { |
| if (isHot && isHotBlockNthPercentile(PercentileCutoff, &BB, &BFI)) |
| return true; |
| if (!isHot && !isColdBlockNthPercentile(PercentileCutoff, &BB, &BFI)) |
| return false; |
| } |
| return !isHot; |
| } |
| |
| // Like isFunctionHotInCallGraph but for a given cutoff. |
| bool ProfileSummaryInfo::isFunctionHotInCallGraphNthPercentile( |
| int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const { |
| return isFunctionHotOrColdInCallGraphNthPercentile<true>( |
| PercentileCutoff, F, BFI); |
| } |
| |
| bool ProfileSummaryInfo::isFunctionColdInCallGraphNthPercentile( |
| int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const { |
| return isFunctionHotOrColdInCallGraphNthPercentile<false>( |
| PercentileCutoff, F, BFI); |
| } |
| |
| /// Returns true if the function's entry is a cold. If it returns false, it |
| /// either means it is not cold or it is unknown whether it is cold or not (for |
| /// example, no profile data is available). |
| bool ProfileSummaryInfo::isFunctionEntryCold(const Function *F) const { |
| if (!F) |
| return false; |
| if (F->hasFnAttribute(Attribute::Cold)) |
| return true; |
| if (!hasProfileSummary()) |
| return false; |
| auto FunctionCount = F->getEntryCount(); |
| // FIXME: The heuristic used below for determining coldness is based on |
| // preliminary SPEC tuning for inliner. This will eventually be a |
| // convenience method that calls isHotCount. |
| return FunctionCount && isColdCount(FunctionCount.getCount()); |
| } |
| |
| /// Compute the hot and cold thresholds. |
| void ProfileSummaryInfo::computeThresholds() { |
| auto &DetailedSummary = Summary->getDetailedSummary(); |
| auto &HotEntry = ProfileSummaryBuilder::getEntryForPercentile( |
| DetailedSummary, ProfileSummaryCutoffHot); |
| HotCountThreshold = HotEntry.MinCount; |
| if (ProfileSummaryHotCount.getNumOccurrences() > 0) |
| HotCountThreshold = ProfileSummaryHotCount; |
| auto &ColdEntry = ProfileSummaryBuilder::getEntryForPercentile( |
| DetailedSummary, ProfileSummaryCutoffCold); |
| ColdCountThreshold = ColdEntry.MinCount; |
| if (ProfileSummaryColdCount.getNumOccurrences() > 0) |
| ColdCountThreshold = ProfileSummaryColdCount; |
| assert(ColdCountThreshold <= HotCountThreshold && |
| "Cold count threshold cannot exceed hot count threshold!"); |
| if (!hasPartialSampleProfile() || !ScalePartialSampleProfileWorkingSetSize) { |
| HasHugeWorkingSetSize = |
| HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold; |
| HasLargeWorkingSetSize = |
| HotEntry.NumCounts > ProfileSummaryLargeWorkingSetSizeThreshold; |
| } else { |
| // Scale the working set size of the partial sample profile to reflect the |
| // size of the program being compiled. |
| double PartialProfileRatio = Summary->getPartialProfileRatio(); |
| uint64_t ScaledHotEntryNumCounts = |
| static_cast<uint64_t>(HotEntry.NumCounts * PartialProfileRatio * |
| PartialSampleProfileWorkingSetSizeScaleFactor); |
| HasHugeWorkingSetSize = |
| ScaledHotEntryNumCounts > ProfileSummaryHugeWorkingSetSizeThreshold; |
| HasLargeWorkingSetSize = |
| ScaledHotEntryNumCounts > ProfileSummaryLargeWorkingSetSizeThreshold; |
| } |
| } |
| |
| Optional<uint64_t> |
| ProfileSummaryInfo::computeThreshold(int PercentileCutoff) const { |
| if (!hasProfileSummary()) |
| return None; |
| auto iter = ThresholdCache.find(PercentileCutoff); |
| if (iter != ThresholdCache.end()) { |
| return iter->second; |
| } |
| auto &DetailedSummary = Summary->getDetailedSummary(); |
| auto &Entry = ProfileSummaryBuilder::getEntryForPercentile(DetailedSummary, |
| PercentileCutoff); |
| uint64_t CountThreshold = Entry.MinCount; |
| ThresholdCache[PercentileCutoff] = CountThreshold; |
| return CountThreshold; |
| } |
| |
| bool ProfileSummaryInfo::hasHugeWorkingSetSize() const { |
| return HasHugeWorkingSetSize && HasHugeWorkingSetSize.getValue(); |
| } |
| |
| bool ProfileSummaryInfo::hasLargeWorkingSetSize() const { |
| return HasLargeWorkingSetSize && HasLargeWorkingSetSize.getValue(); |
| } |
| |
| bool ProfileSummaryInfo::isHotCount(uint64_t C) const { |
| return HotCountThreshold && C >= HotCountThreshold.getValue(); |
| } |
| |
| bool ProfileSummaryInfo::isColdCount(uint64_t C) const { |
| return ColdCountThreshold && C <= ColdCountThreshold.getValue(); |
| } |
| |
| template <bool isHot> |
| bool ProfileSummaryInfo::isHotOrColdCountNthPercentile(int PercentileCutoff, |
| uint64_t C) const { |
| auto CountThreshold = computeThreshold(PercentileCutoff); |
| if (isHot) |
| return CountThreshold && C >= CountThreshold.getValue(); |
| else |
| return CountThreshold && C <= CountThreshold.getValue(); |
| } |
| |
| bool ProfileSummaryInfo::isHotCountNthPercentile(int PercentileCutoff, |
| uint64_t C) const { |
| return isHotOrColdCountNthPercentile<true>(PercentileCutoff, C); |
| } |
| |
| bool ProfileSummaryInfo::isColdCountNthPercentile(int PercentileCutoff, |
| uint64_t C) const { |
| return isHotOrColdCountNthPercentile<false>(PercentileCutoff, C); |
| } |
| |
| uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() const { |
| return HotCountThreshold ? HotCountThreshold.getValue() : UINT64_MAX; |
| } |
| |
| uint64_t ProfileSummaryInfo::getOrCompColdCountThreshold() const { |
| return ColdCountThreshold ? ColdCountThreshold.getValue() : 0; |
| } |
| |
| bool ProfileSummaryInfo::isHotBlock(const BasicBlock *BB, |
| BlockFrequencyInfo *BFI) const { |
| auto Count = BFI->getBlockProfileCount(BB); |
| return Count && isHotCount(*Count); |
| } |
| |
| bool ProfileSummaryInfo::isColdBlock(const BasicBlock *BB, |
| BlockFrequencyInfo *BFI) const { |
| auto Count = BFI->getBlockProfileCount(BB); |
| return Count && isColdCount(*Count); |
| } |
| |
| template <bool isHot> |
| bool ProfileSummaryInfo::isHotOrColdBlockNthPercentile( |
| int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const { |
| auto Count = BFI->getBlockProfileCount(BB); |
| if (isHot) |
| return Count && isHotCountNthPercentile(PercentileCutoff, *Count); |
| else |
| return Count && isColdCountNthPercentile(PercentileCutoff, *Count); |
| } |
| |
| bool ProfileSummaryInfo::isHotBlockNthPercentile( |
| int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const { |
| return isHotOrColdBlockNthPercentile<true>(PercentileCutoff, BB, BFI); |
| } |
| |
| bool ProfileSummaryInfo::isColdBlockNthPercentile( |
| int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const { |
| return isHotOrColdBlockNthPercentile<false>(PercentileCutoff, BB, BFI); |
| } |
| |
| bool ProfileSummaryInfo::isHotCallSite(const CallBase &CB, |
| BlockFrequencyInfo *BFI) const { |
| auto C = getProfileCount(CB, BFI); |
| return C && isHotCount(*C); |
| } |
| |
| bool ProfileSummaryInfo::isColdCallSite(const CallBase &CB, |
| BlockFrequencyInfo *BFI) const { |
| auto C = getProfileCount(CB, BFI); |
| if (C) |
| return isColdCount(*C); |
| |
| // In SamplePGO, if the caller has been sampled, and there is no profile |
| // annotated on the callsite, we consider the callsite as cold. |
| return hasSampleProfile() && CB.getCaller()->hasProfileData(); |
| } |
| |
| bool ProfileSummaryInfo::hasPartialSampleProfile() const { |
| return hasProfileSummary() && |
| Summary->getKind() == ProfileSummary::PSK_Sample && |
| (PartialProfile || Summary->isPartialProfile()); |
| } |
| |
| INITIALIZE_PASS(ProfileSummaryInfoWrapperPass, "profile-summary-info", |
| "Profile summary info", false, true) |
| |
| ProfileSummaryInfoWrapperPass::ProfileSummaryInfoWrapperPass() |
| : ImmutablePass(ID) { |
| initializeProfileSummaryInfoWrapperPassPass(*PassRegistry::getPassRegistry()); |
| } |
| |
| bool ProfileSummaryInfoWrapperPass::doInitialization(Module &M) { |
| PSI.reset(new ProfileSummaryInfo(M)); |
| return false; |
| } |
| |
| bool ProfileSummaryInfoWrapperPass::doFinalization(Module &M) { |
| PSI.reset(); |
| return false; |
| } |
| |
| AnalysisKey ProfileSummaryAnalysis::Key; |
| ProfileSummaryInfo ProfileSummaryAnalysis::run(Module &M, |
| ModuleAnalysisManager &) { |
| return ProfileSummaryInfo(M); |
| } |
| |
| PreservedAnalyses ProfileSummaryPrinterPass::run(Module &M, |
| ModuleAnalysisManager &AM) { |
| ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(M); |
| |
| OS << "Functions in " << M.getName() << " with hot/cold annotations: \n"; |
| for (auto &F : M) { |
| OS << F.getName(); |
| if (PSI.isFunctionEntryHot(&F)) |
| OS << " :hot entry "; |
| else if (PSI.isFunctionEntryCold(&F)) |
| OS << " :cold entry "; |
| OS << "\n"; |
| } |
| return PreservedAnalyses::all(); |
| } |
| |
| char ProfileSummaryInfoWrapperPass::ID = 0; |