llvm/lib/Transforms/Utils/SampleProfileLoaderBaseUtil.cpp - llvm-project - Git at Google

 //===- SampleProfileLoaderBaseUtil.cpp - Profile loader Util func ---------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the SampleProfileLoader base utility functions.
 //
 //===----------------------------------------------------------------------===//

 #include "llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h"
 #include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"

 namespace llvm {

 cl::opt<unsigned> SampleProfileMaxPropagateIterations(
     "sample-profile-max-propagate-iterations", cl::init(100),
     cl::desc("Maximum number of iterations to go through when propagating "
              "sample block/edge weights through the CFG."));

 cl::opt<unsigned> SampleProfileRecordCoverage(
     "sample-profile-check-record-coverage", cl::init(0), cl::value_desc("N"),
     cl::desc("Emit a warning if less than N% of records in the input profile "
              "are matched to the IR."));

 cl::opt<unsigned> SampleProfileSampleCoverage(
     "sample-profile-check-sample-coverage", cl::init(0), cl::value_desc("N"),
     cl::desc("Emit a warning if less than N% of samples in the input profile "
              "are matched to the IR."));

 cl::opt<bool> NoWarnSampleUnused(
     "no-warn-sample-unused", cl::init(false), cl::Hidden,
     cl::desc("Use this option to turn off/on warnings about function with "
              "samples but without debug information to use those samples. "));

 cl::opt<bool> SampleProfileUseProfi(
     "sample-profile-use-profi", cl::Hidden,
     cl::desc("Use profi to infer block and edge counts."));

 namespace sampleprofutil {

 /// Return true if the given callsite is hot wrt to hot cutoff threshold.
 ///
 /// Functions that were inlined in the original binary will be represented
 /// in the inline stack in the sample profile. If the profile shows that
 /// the original inline decision was "good" (i.e., the callsite is executed
 /// frequently), then we will recreate the inline decision and apply the
 /// profile from the inlined callsite.
 ///
 /// To decide whether an inlined callsite is hot, we compare the callsite
 /// sample count with the hot cutoff computed by ProfileSummaryInfo, it is
 /// regarded as hot if the count is above the cutoff value.
 ///
 /// When ProfileAccurateForSymsInList is enabled and profile symbol list
 /// is present, functions in the profile symbol list but without profile will
 /// be regarded as cold and much less inlining will happen in CGSCC inlining
 /// pass, so we tend to lower the hot criteria here to allow more early
 /// inlining to happen for warm callsites and it is helpful for performance.
 bool callsiteIsHot(const FunctionSamples *CallsiteFS, ProfileSummaryInfo *PSI,
                    bool ProfAccForSymsInList) {
   if (!CallsiteFS)
     return false; // The callsite was not inlined in the original binary.

   assert(PSI && "PSI is expected to be non null");
   uint64_t CallsiteTotalSamples = CallsiteFS->getTotalSamples();
   if (ProfAccForSymsInList)
     return !PSI->isColdCount(CallsiteTotalSamples);
   else
     return PSI->isHotCount(CallsiteTotalSamples);
 }

 /// Mark as used the sample record for the given function samples at
 /// (LineOffset, Discriminator).
 ///
 /// \returns true if this is the first time we mark the given record.
 bool SampleCoverageTracker::markSamplesUsed(const FunctionSamples *FS,
                                             uint32_t LineOffset,
                                             uint32_t Discriminator,
                                             uint64_t Samples) {
   LineLocation Loc(LineOffset, Discriminator);
   unsigned &Count = SampleCoverage[FS][Loc];
   bool FirstTime = (++Count == 1);
   if (FirstTime)
     TotalUsedSamples += Samples;
   return FirstTime;
 }

 /// Return the number of sample records that were applied from this profile.
 ///
 /// This count does not include records from cold inlined callsites.
 unsigned
 SampleCoverageTracker::countUsedRecords(const FunctionSamples *FS,
                                         ProfileSummaryInfo *PSI) const {
   auto I = SampleCoverage.find(FS);

   // The size of the coverage map for FS represents the number of records
   // that were marked used at least once.
   unsigned Count = (I != SampleCoverage.end()) ? I->second.size() : 0;

   // If there are inlined callsites in this function, count the samples found
   // in the respective bodies. However, do not bother counting callees with 0
   // total samples, these are callees that were never invoked at runtime.
   for (const auto &I : FS->getCallsiteSamples())
     for (const auto &J : I.second) {
       const FunctionSamples *CalleeSamples = &J.second;
       if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
         Count += countUsedRecords(CalleeSamples, PSI);
     }

   return Count;
 }

 /// Return the number of sample records in the body of this profile.
 ///
 /// This count does not include records from cold inlined callsites.
 unsigned
 SampleCoverageTracker::countBodyRecords(const FunctionSamples *FS,
                                         ProfileSummaryInfo *PSI) const {
   unsigned Count = FS->getBodySamples().size();

   // Only count records in hot callsites.
   for (const auto &I : FS->getCallsiteSamples())
     for (const auto &J : I.second) {
       const FunctionSamples *CalleeSamples = &J.second;
       if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
         Count += countBodyRecords(CalleeSamples, PSI);
     }

   return Count;
 }

 /// Return the number of samples collected in the body of this profile.
 ///
 /// This count does not include samples from cold inlined callsites.
 uint64_t
 SampleCoverageTracker::countBodySamples(const FunctionSamples *FS,
                                         ProfileSummaryInfo *PSI) const {
   uint64_t Total = 0;
   for (const auto &I : FS->getBodySamples())
     Total += I.second.getSamples();

   // Only count samples in hot callsites.
   for (const auto &I : FS->getCallsiteSamples())
     for (const auto &J : I.second) {
       const FunctionSamples *CalleeSamples = &J.second;
       if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
         Total += countBodySamples(CalleeSamples, PSI);
     }

   return Total;
 }

 /// Return the fraction of sample records used in this profile.
 ///
 /// The returned value is an unsigned integer in the range 0-100 indicating
 /// the percentage of sample records that were used while applying this
 /// profile to the associated function.
 unsigned SampleCoverageTracker::computeCoverage(unsigned Used,
                                                 unsigned Total) const {
   assert(Used <= Total &&
          "number of used records cannot exceed the total number of records");
   return Total > 0 ? Used * 100 / Total : 100;
 }

 /// Create a global variable to flag FSDiscriminators are used.
 void createFSDiscriminatorVariable(Module *M) {
   const char *FSDiscriminatorVar = "__llvm_fs_discriminator__";
   if (M->getGlobalVariable(FSDiscriminatorVar))
     return;

   auto &Context = M->getContext();
   // Place this variable to llvm.used so it won't be GC'ed.
   appendToUsed(*M, {new GlobalVariable(*M, Type::getInt1Ty(Context), true,
                                        GlobalValue::WeakODRLinkage,
                                        ConstantInt::getTrue(Context),
                                        FSDiscriminatorVar)});
 }

 } // end of namespace sampleprofutil
 } // end of namespace llvm
	//===- SampleProfileLoaderBaseUtil.cpp - Profile loader Util func ---------===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	//
	// This file implements the SampleProfileLoader base utility functions.
	//
	//===----------------------------------------------------------------------===//

	#include "llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h"
	#include "llvm/Analysis/ProfileSummaryInfo.h"
	#include "llvm/IR/Constants.h"
	#include "llvm/IR/Module.h"
	#include "llvm/Transforms/Utils/ModuleUtils.h"

	namespace llvm {

	cl::opt<unsigned> SampleProfileMaxPropagateIterations(
	"sample-profile-max-propagate-iterations", cl::init(100),
	cl::desc("Maximum number of iterations to go through when propagating "
	"sample block/edge weights through the CFG."));

	cl::opt<unsigned> SampleProfileRecordCoverage(
	"sample-profile-check-record-coverage", cl::init(0), cl::value_desc("N"),
	cl::desc("Emit a warning if less than N% of records in the input profile "
	"are matched to the IR."));

	cl::opt<unsigned> SampleProfileSampleCoverage(
	"sample-profile-check-sample-coverage", cl::init(0), cl::value_desc("N"),
	cl::desc("Emit a warning if less than N% of samples in the input profile "
	"are matched to the IR."));

	cl::opt<bool> NoWarnSampleUnused(
	"no-warn-sample-unused", cl::init(false), cl::Hidden,
	cl::desc("Use this option to turn off/on warnings about function with "
	"samples but without debug information to use those samples. "));

	cl::opt<bool> SampleProfileUseProfi(
	"sample-profile-use-profi", cl::Hidden,
	cl::desc("Use profi to infer block and edge counts."));

	namespace sampleprofutil {

	/// Return true if the given callsite is hot wrt to hot cutoff threshold.
	///
	/// Functions that were inlined in the original binary will be represented
	/// in the inline stack in the sample profile. If the profile shows that
	/// the original inline decision was "good" (i.e., the callsite is executed
	/// frequently), then we will recreate the inline decision and apply the
	/// profile from the inlined callsite.
	///
	/// To decide whether an inlined callsite is hot, we compare the callsite
	/// sample count with the hot cutoff computed by ProfileSummaryInfo, it is
	/// regarded as hot if the count is above the cutoff value.
	///
	/// When ProfileAccurateForSymsInList is enabled and profile symbol list
	/// is present, functions in the profile symbol list but without profile will
	/// be regarded as cold and much less inlining will happen in CGSCC inlining
	/// pass, so we tend to lower the hot criteria here to allow more early
	/// inlining to happen for warm callsites and it is helpful for performance.
	bool callsiteIsHot(const FunctionSamples CallsiteFS, ProfileSummaryInfo PSI,
	bool ProfAccForSymsInList) {
	if (!CallsiteFS)
	return false; // The callsite was not inlined in the original binary.

	assert(PSI && "PSI is expected to be non null");
	uint64_t CallsiteTotalSamples = CallsiteFS->getTotalSamples();
	if (ProfAccForSymsInList)
	return !PSI->isColdCount(CallsiteTotalSamples);
	else
	return PSI->isHotCount(CallsiteTotalSamples);
	}

	/// Mark as used the sample record for the given function samples at
	/// (LineOffset, Discriminator).
	///
	/// \returns true if this is the first time we mark the given record.
	bool SampleCoverageTracker::markSamplesUsed(const FunctionSamples *FS,
	uint32_t LineOffset,
	uint32_t Discriminator,
	uint64_t Samples) {
	LineLocation Loc(LineOffset, Discriminator);
	unsigned &Count = SampleCoverage[FS][Loc];
	bool FirstTime = (++Count == 1);
	if (FirstTime)
	TotalUsedSamples += Samples;
	return FirstTime;
	}

	/// Return the number of sample records that were applied from this profile.
	///
	/// This count does not include records from cold inlined callsites.
	unsigned
	SampleCoverageTracker::countUsedRecords(const FunctionSamples *FS,
	ProfileSummaryInfo *PSI) const {
	auto I = SampleCoverage.find(FS);

	// The size of the coverage map for FS represents the number of records
	// that were marked used at least once.
	unsigned Count = (I != SampleCoverage.end()) ? I->second.size() : 0;

	// If there are inlined callsites in this function, count the samples found
	// in the respective bodies. However, do not bother counting callees with 0
	// total samples, these are callees that were never invoked at runtime.
	for (const auto &I : FS->getCallsiteSamples())
	for (const auto &J : I.second) {
	const FunctionSamples *CalleeSamples = &J.second;
	if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
	Count += countUsedRecords(CalleeSamples, PSI);
	}

	return Count;
	}

	/// Return the number of sample records in the body of this profile.
	///
	/// This count does not include records from cold inlined callsites.
	unsigned
	SampleCoverageTracker::countBodyRecords(const FunctionSamples *FS,
	ProfileSummaryInfo *PSI) const {
	unsigned Count = FS->getBodySamples().size();

	// Only count records in hot callsites.
	for (const auto &I : FS->getCallsiteSamples())
	for (const auto &J : I.second) {
	const FunctionSamples *CalleeSamples = &J.second;
	if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
	Count += countBodyRecords(CalleeSamples, PSI);
	}

	return Count;
	}

	/// Return the number of samples collected in the body of this profile.
	///
	/// This count does not include samples from cold inlined callsites.
	uint64_t
	SampleCoverageTracker::countBodySamples(const FunctionSamples *FS,
	ProfileSummaryInfo *PSI) const {
	uint64_t Total = 0;
	for (const auto &I : FS->getBodySamples())
	Total += I.second.getSamples();

	// Only count samples in hot callsites.
	for (const auto &I : FS->getCallsiteSamples())
	for (const auto &J : I.second) {
	const FunctionSamples *CalleeSamples = &J.second;
	if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
	Total += countBodySamples(CalleeSamples, PSI);
	}

	return Total;
	}

	/// Return the fraction of sample records used in this profile.
	///
	/// The returned value is an unsigned integer in the range 0-100 indicating
	/// the percentage of sample records that were used while applying this
	/// profile to the associated function.
	unsigned SampleCoverageTracker::computeCoverage(unsigned Used,
	unsigned Total) const {
	assert(Used <= Total &&
	"number of used records cannot exceed the total number of records");
	return Total > 0 ? Used * 100 / Total : 100;
	}

	/// Create a global variable to flag FSDiscriminators are used.
	void createFSDiscriminatorVariable(Module *M) {
	const char *FSDiscriminatorVar = "__llvm_fs_discriminator__";
	if (M->getGlobalVariable(FSDiscriminatorVar))
	return;

	auto &Context = M->getContext();
	// Place this variable to llvm.used so it won't be GC'ed.
	appendToUsed(M, {new GlobalVariable(M, Type::getInt1Ty(Context), true,
	GlobalValue::WeakODRLinkage,
	ConstantInt::getTrue(Context),
	FSDiscriminatorVar)});
	}

	} // end of namespace sampleprofutil
	} // end of namespace llvm