| //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===// |
| // |
| // The LLVM Compiler Infrastructure |
| // |
| // This file is distributed under the University of Illinois Open Source |
| // License. See LICENSE.TXT for details. |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This pass lowers instrprof_increment intrinsics emitted by a frontend for |
| // profiling. It also builds the data structures and initialization code needed |
| // for updating execution counts and emitting the profile at runtime. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "llvm/Transforms/Instrumentation.h" |
| |
| #include "llvm/ADT/Triple.h" |
| #include "llvm/IR/IRBuilder.h" |
| #include "llvm/IR/IntrinsicInst.h" |
| #include "llvm/IR/Module.h" |
| #include "llvm/Transforms/Utils/ModuleUtils.h" |
| |
| using namespace llvm; |
| |
| #define DEBUG_TYPE "instrprof" |
| |
| namespace { |
| |
| class InstrProfiling : public ModulePass { |
| public: |
| static char ID; |
| |
| InstrProfiling() : ModulePass(ID) {} |
| |
| InstrProfiling(const InstrProfOptions &Options) |
| : ModulePass(ID), Options(Options) {} |
| |
| const char *getPassName() const override { |
| return "Frontend instrumentation-based coverage lowering"; |
| } |
| |
| bool runOnModule(Module &M) override; |
| |
| void getAnalysisUsage(AnalysisUsage &AU) const override { |
| AU.setPreservesCFG(); |
| } |
| |
| private: |
| InstrProfOptions Options; |
| Module *M; |
| DenseMap<GlobalVariable *, GlobalVariable *> RegionCounters; |
| std::vector<Value *> UsedVars; |
| |
| bool isMachO() const { |
| return Triple(M->getTargetTriple()).isOSBinFormatMachO(); |
| } |
| |
| /// Get the section name for the counter variables. |
| StringRef getCountersSection() const { |
| return isMachO() ? "__DATA,__llvm_prf_cnts" : "__llvm_prf_cnts"; |
| } |
| |
| /// Get the section name for the name variables. |
| StringRef getNameSection() const { |
| return isMachO() ? "__DATA,__llvm_prf_names" : "__llvm_prf_names"; |
| } |
| |
| /// Get the section name for the profile data variables. |
| StringRef getDataSection() const { |
| return isMachO() ? "__DATA,__llvm_prf_data" : "__llvm_prf_data"; |
| } |
| |
| /// Get the section name for the coverage mapping data. |
| StringRef getCoverageSection() const { |
| return isMachO() ? "__DATA,__llvm_covmap" : "__llvm_covmap"; |
| } |
| |
| /// Replace instrprof_increment with an increment of the appropriate value. |
| void lowerIncrement(InstrProfIncrementInst *Inc); |
| |
| /// Set up the section and uses for coverage data and its references. |
| void lowerCoverageData(GlobalVariable *CoverageData); |
| |
| /// Get the region counters for an increment, creating them if necessary. |
| /// |
| /// If the counter array doesn't yet exist, the profile data variables |
| /// referring to them will also be created. |
| GlobalVariable *getOrCreateRegionCounters(InstrProfIncrementInst *Inc); |
| |
| /// Emit runtime registration functions for each profile data variable. |
| void emitRegistration(); |
| |
| /// Emit the necessary plumbing to pull in the runtime initialization. |
| void emitRuntimeHook(); |
| |
| /// Add uses of our data variables and runtime hook. |
| void emitUses(); |
| |
| /// Create a static initializer for our data, on platforms that need it, |
| /// and for any profile output file that was specified. |
| void emitInitialization(); |
| }; |
| |
| } // anonymous namespace |
| |
| char InstrProfiling::ID = 0; |
| INITIALIZE_PASS(InstrProfiling, "instrprof", |
| "Frontend instrumentation-based coverage lowering.", false, |
| false) |
| |
| ModulePass *llvm::createInstrProfilingPass(const InstrProfOptions &Options) { |
| return new InstrProfiling(Options); |
| } |
| |
| bool InstrProfiling::runOnModule(Module &M) { |
| bool MadeChange = false; |
| |
| this->M = &M; |
| RegionCounters.clear(); |
| UsedVars.clear(); |
| |
| for (Function &F : M) |
| for (BasicBlock &BB : F) |
| for (auto I = BB.begin(), E = BB.end(); I != E;) |
| if (auto *Inc = dyn_cast<InstrProfIncrementInst>(I++)) { |
| lowerIncrement(Inc); |
| MadeChange = true; |
| } |
| if (GlobalVariable *Coverage = M.getNamedGlobal("__llvm_coverage_mapping")) { |
| lowerCoverageData(Coverage); |
| MadeChange = true; |
| } |
| if (!MadeChange) |
| return false; |
| |
| emitRegistration(); |
| emitRuntimeHook(); |
| emitUses(); |
| emitInitialization(); |
| return true; |
| } |
| |
| void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) { |
| GlobalVariable *Counters = getOrCreateRegionCounters(Inc); |
| |
| IRBuilder<> Builder(Inc->getParent(), *Inc); |
| uint64_t Index = Inc->getIndex()->getZExtValue(); |
| Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index); |
| Value *Count = Builder.CreateLoad(Addr, "pgocount"); |
| Count = Builder.CreateAdd(Count, Builder.getInt64(1)); |
| Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr)); |
| Inc->eraseFromParent(); |
| } |
| |
| void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageData) { |
| CoverageData->setSection(getCoverageSection()); |
| CoverageData->setAlignment(8); |
| |
| Constant *Init = CoverageData->getInitializer(); |
| // We're expecting { i32, i32, i32, i32, [n x { i8*, i32, i32 }], [m x i8] } |
| // for some C. If not, the frontend's given us something broken. |
| assert(Init->getNumOperands() == 6 && "bad number of fields in coverage map"); |
| assert(isa<ConstantArray>(Init->getAggregateElement(4)) && |
| "invalid function list in coverage map"); |
| ConstantArray *Records = cast<ConstantArray>(Init->getAggregateElement(4)); |
| for (unsigned I = 0, E = Records->getNumOperands(); I < E; ++I) { |
| Constant *Record = Records->getOperand(I); |
| Value *V = const_cast<Value *>(Record->getOperand(0))->stripPointerCasts(); |
| |
| assert(isa<GlobalVariable>(V) && "Missing reference to function name"); |
| GlobalVariable *Name = cast<GlobalVariable>(V); |
| |
| // If we have region counters for this name, we've already handled it. |
| auto It = RegionCounters.find(Name); |
| if (It != RegionCounters.end()) |
| continue; |
| |
| // Move the name variable to the right section. |
| Name->setSection(getNameSection()); |
| Name->setAlignment(1); |
| } |
| } |
| |
| /// Get the name of a profiling variable for a particular function. |
| static std::string getVarName(InstrProfIncrementInst *Inc, StringRef VarName) { |
| auto *Arr = cast<ConstantDataArray>(Inc->getName()->getInitializer()); |
| StringRef Name = Arr->isCString() ? Arr->getAsCString() : Arr->getAsString(); |
| return ("__llvm_profile_" + VarName + "_" + Name).str(); |
| } |
| |
| GlobalVariable * |
| InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) { |
| GlobalVariable *Name = Inc->getName(); |
| auto It = RegionCounters.find(Name); |
| if (It != RegionCounters.end()) |
| return It->second; |
| |
| // Move the name variable to the right section. Make sure it is placed in the |
| // same comdat as its associated function. Otherwise, we may get multiple |
| // counters for the same function in certain cases. |
| Function *Fn = Inc->getParent()->getParent(); |
| Name->setSection(getNameSection()); |
| Name->setAlignment(1); |
| Name->setComdat(Fn->getComdat()); |
| |
| uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); |
| LLVMContext &Ctx = M->getContext(); |
| ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); |
| |
| // Create the counters variable. |
| auto *Counters = new GlobalVariable(*M, CounterTy, false, Name->getLinkage(), |
| Constant::getNullValue(CounterTy), |
| getVarName(Inc, "counters")); |
| Counters->setVisibility(Name->getVisibility()); |
| Counters->setSection(getCountersSection()); |
| Counters->setAlignment(8); |
| Counters->setComdat(Fn->getComdat()); |
| |
| RegionCounters[Inc->getName()] = Counters; |
| |
| // Create data variable. |
| auto *NameArrayTy = Name->getType()->getPointerElementType(); |
| auto *Int32Ty = Type::getInt32Ty(Ctx); |
| auto *Int64Ty = Type::getInt64Ty(Ctx); |
| auto *Int8PtrTy = Type::getInt8PtrTy(Ctx); |
| auto *Int64PtrTy = Type::getInt64PtrTy(Ctx); |
| |
| Type *DataTypes[] = {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int64PtrTy}; |
| auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes)); |
| Constant *DataVals[] = { |
| ConstantInt::get(Int32Ty, NameArrayTy->getArrayNumElements()), |
| ConstantInt::get(Int32Ty, NumCounters), |
| ConstantInt::get(Int64Ty, Inc->getHash()->getZExtValue()), |
| ConstantExpr::getBitCast(Name, Int8PtrTy), |
| ConstantExpr::getBitCast(Counters, Int64PtrTy)}; |
| auto *Data = new GlobalVariable(*M, DataTy, true, Name->getLinkage(), |
| ConstantStruct::get(DataTy, DataVals), |
| getVarName(Inc, "data")); |
| Data->setVisibility(Name->getVisibility()); |
| Data->setSection(getDataSection()); |
| Data->setAlignment(8); |
| Data->setComdat(Fn->getComdat()); |
| |
| // Mark the data variable as used so that it isn't stripped out. |
| UsedVars.push_back(Data); |
| |
| return Counters; |
| } |
| |
| void InstrProfiling::emitRegistration() { |
| // Don't do this for Darwin. compiler-rt uses linker magic. |
| if (Triple(M->getTargetTriple()).isOSDarwin()) |
| return; |
| |
| // Construct the function. |
| auto *VoidTy = Type::getVoidTy(M->getContext()); |
| auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext()); |
| auto *RegisterFTy = FunctionType::get(VoidTy, false); |
| auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage, |
| "__llvm_profile_register_functions", M); |
| RegisterF->setUnnamedAddr(true); |
| if (Options.NoRedZone) |
| RegisterF->addFnAttr(Attribute::NoRedZone); |
| |
| auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false); |
| auto *RuntimeRegisterF = |
| Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage, |
| "__llvm_profile_register_function", M); |
| |
| IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF)); |
| for (Value *Data : UsedVars) |
| IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy)); |
| IRB.CreateRetVoid(); |
| } |
| |
| void InstrProfiling::emitRuntimeHook() { |
| const char *const RuntimeVarName = "__llvm_profile_runtime"; |
| const char *const RuntimeUserName = "__llvm_profile_runtime_user"; |
| |
| // If the module's provided its own runtime, we don't need to do anything. |
| if (M->getGlobalVariable(RuntimeVarName)) |
| return; |
| |
| // Declare an external variable that will pull in the runtime initialization. |
| auto *Int32Ty = Type::getInt32Ty(M->getContext()); |
| auto *Var = |
| new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage, |
| nullptr, RuntimeVarName); |
| |
| // Make a function that uses it. |
| auto *User = |
| Function::Create(FunctionType::get(Int32Ty, false), |
| GlobalValue::LinkOnceODRLinkage, RuntimeUserName, M); |
| User->addFnAttr(Attribute::NoInline); |
| if (Options.NoRedZone) |
| User->addFnAttr(Attribute::NoRedZone); |
| User->setVisibility(GlobalValue::HiddenVisibility); |
| |
| IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User)); |
| auto *Load = IRB.CreateLoad(Var); |
| IRB.CreateRet(Load); |
| |
| // Mark the user variable as used so that it isn't stripped out. |
| UsedVars.push_back(User); |
| } |
| |
| void InstrProfiling::emitUses() { |
| if (UsedVars.empty()) |
| return; |
| |
| GlobalVariable *LLVMUsed = M->getGlobalVariable("llvm.used"); |
| std::vector<Constant *> MergedVars; |
| if (LLVMUsed) { |
| // Collect the existing members of llvm.used. |
| ConstantArray *Inits = cast<ConstantArray>(LLVMUsed->getInitializer()); |
| for (unsigned I = 0, E = Inits->getNumOperands(); I != E; ++I) |
| MergedVars.push_back(Inits->getOperand(I)); |
| LLVMUsed->eraseFromParent(); |
| } |
| |
| Type *i8PTy = Type::getInt8PtrTy(M->getContext()); |
| // Add uses for our data. |
| for (auto *Value : UsedVars) |
| MergedVars.push_back( |
| ConstantExpr::getBitCast(cast<Constant>(Value), i8PTy)); |
| |
| // Recreate llvm.used. |
| ArrayType *ATy = ArrayType::get(i8PTy, MergedVars.size()); |
| LLVMUsed = |
| new GlobalVariable(*M, ATy, false, GlobalValue::AppendingLinkage, |
| ConstantArray::get(ATy, MergedVars), "llvm.used"); |
| |
| LLVMUsed->setSection("llvm.metadata"); |
| } |
| |
| void InstrProfiling::emitInitialization() { |
| std::string InstrProfileOutput = Options.InstrProfileOutput; |
| |
| Constant *RegisterF = M->getFunction("__llvm_profile_register_functions"); |
| if (!RegisterF && InstrProfileOutput.empty()) |
| return; |
| |
| // Create the initialization function. |
| auto *VoidTy = Type::getVoidTy(M->getContext()); |
| auto *F = |
| Function::Create(FunctionType::get(VoidTy, false), |
| GlobalValue::InternalLinkage, "__llvm_profile_init", M); |
| F->setUnnamedAddr(true); |
| F->addFnAttr(Attribute::NoInline); |
| if (Options.NoRedZone) |
| F->addFnAttr(Attribute::NoRedZone); |
| |
| // Add the basic block and the necessary calls. |
| IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F)); |
| if (RegisterF) |
| IRB.CreateCall(RegisterF, {}); |
| if (!InstrProfileOutput.empty()) { |
| auto *Int8PtrTy = Type::getInt8PtrTy(M->getContext()); |
| auto *SetNameTy = FunctionType::get(VoidTy, Int8PtrTy, false); |
| auto *SetNameF = |
| Function::Create(SetNameTy, GlobalValue::ExternalLinkage, |
| "__llvm_profile_override_default_filename", M); |
| |
| // Create variable for profile name. |
| Constant *ProfileNameConst = |
| ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true); |
| GlobalVariable *ProfileName = |
| new GlobalVariable(*M, ProfileNameConst->getType(), true, |
| GlobalValue::PrivateLinkage, ProfileNameConst); |
| |
| IRB.CreateCall(SetNameF, IRB.CreatePointerCast(ProfileName, Int8PtrTy)); |
| } |
| IRB.CreateRetVoid(); |
| |
| appendToGlobalCtors(*M, F, 0); |
| } |