| //===- PseudoProbe.cpp - Pseudo Probe Helpers -----------------------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file implements the helpers to manipulate pseudo probe IR intrinsic |
| // calls. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "llvm/IR/PseudoProbe.h" |
| #include "llvm/IR/DebugInfoMetadata.h" |
| #include "llvm/IR/IRBuilder.h" |
| #include "llvm/IR/Instruction.h" |
| #include <unordered_set> |
| |
| using namespace llvm; |
| |
| namespace llvm { |
| |
| Optional<PseudoProbe> extractProbeFromDiscriminator(const Instruction &Inst) { |
| assert(isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst) && |
| "Only call instructions should have pseudo probe encodes as their " |
| "Dwarf discriminators"); |
| if (const DebugLoc &DLoc = Inst.getDebugLoc()) { |
| const DILocation *DIL = DLoc; |
| auto Discriminator = DIL->getDiscriminator(); |
| if (DILocation::isPseudoProbeDiscriminator(Discriminator)) { |
| PseudoProbe Probe; |
| Probe.Id = |
| PseudoProbeDwarfDiscriminator::extractProbeIndex(Discriminator); |
| Probe.Type = |
| PseudoProbeDwarfDiscriminator::extractProbeType(Discriminator); |
| Probe.Attr = |
| PseudoProbeDwarfDiscriminator::extractProbeAttributes(Discriminator); |
| Probe.Factor = |
| PseudoProbeDwarfDiscriminator::extractProbeFactor(Discriminator) / |
| (float)PseudoProbeDwarfDiscriminator::FullDistributionFactor; |
| return Probe; |
| } |
| } |
| return None; |
| } |
| |
| Optional<PseudoProbe> extractProbe(const Instruction &Inst) { |
| if (const auto *II = dyn_cast<PseudoProbeInst>(&Inst)) { |
| PseudoProbe Probe; |
| Probe.Id = II->getIndex()->getZExtValue(); |
| Probe.Type = (uint32_t)PseudoProbeType::Block; |
| Probe.Attr = II->getAttributes()->getZExtValue(); |
| Probe.Factor = II->getFactor()->getZExtValue() / |
| (float)PseudoProbeFullDistributionFactor; |
| return Probe; |
| } |
| |
| if (isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst)) |
| return extractProbeFromDiscriminator(Inst); |
| |
| return None; |
| } |
| |
| void setProbeDistributionFactor(Instruction &Inst, float Factor) { |
| assert(Factor >= 0 && Factor <= 1 && |
| "Distribution factor must be in [0, 1.0]"); |
| if (auto *II = dyn_cast<PseudoProbeInst>(&Inst)) { |
| IRBuilder<> Builder(&Inst); |
| uint64_t IntFactor = PseudoProbeFullDistributionFactor; |
| if (Factor < 1) |
| IntFactor *= Factor; |
| auto OrigFactor = II->getFactor()->getZExtValue(); |
| if (IntFactor != OrigFactor) |
| II->replaceUsesOfWith(II->getFactor(), Builder.getInt64(IntFactor)); |
| } else if (isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst)) { |
| if (const DebugLoc &DLoc = Inst.getDebugLoc()) { |
| const DILocation *DIL = DLoc; |
| auto Discriminator = DIL->getDiscriminator(); |
| if (DILocation::isPseudoProbeDiscriminator(Discriminator)) { |
| auto Index = |
| PseudoProbeDwarfDiscriminator::extractProbeIndex(Discriminator); |
| auto Type = |
| PseudoProbeDwarfDiscriminator::extractProbeType(Discriminator); |
| auto Attr = PseudoProbeDwarfDiscriminator::extractProbeAttributes( |
| Discriminator); |
| // Round small factors to 0 to avoid over-counting. |
| uint32_t IntFactor = |
| PseudoProbeDwarfDiscriminator::FullDistributionFactor; |
| if (Factor < 1) |
| IntFactor *= Factor; |
| uint32_t V = PseudoProbeDwarfDiscriminator::packProbeData( |
| Index, Type, Attr, IntFactor); |
| DIL = DIL->cloneWithDiscriminator(V); |
| Inst.setDebugLoc(DIL); |
| } |
| } |
| } |
| } |
| |
| void addPseudoProbeAttribute(PseudoProbeInst &Inst, |
| PseudoProbeAttributes Attr) { |
| IRBuilder<> Builder(&Inst); |
| uint32_t OldAttr = Inst.getAttributes()->getZExtValue(); |
| uint32_t NewAttr = OldAttr | (uint32_t)Attr; |
| if (OldAttr != NewAttr) |
| Inst.replaceUsesOfWith(Inst.getAttributes(), Builder.getInt32(NewAttr)); |
| } |
| |
| /// A block emptied (i.e., with all instructions moved out of it) won't be |
| /// sampled at run time. In such cases, AutoFDO will be informed of zero samples |
| /// collected for the block. This is not accurate and could lead to misleading |
| /// weights assigned for the block. A way to mitigate that is to treat such |
| /// block as having unknown counts in the AutoFDO profile loader and allow the |
| /// counts inference tool a chance to calculate a relatively reasonable weight |
| /// for it. This can be done by moving all pseudo probes in the emptied block |
| /// i.e, /c From, to before /c To and tag them dangling. Note that this is |
| /// not needed for dead blocks which really have a zero weight. It's per |
| /// transforms to decide whether to call this function or not. |
| bool moveAndDanglePseudoProbes(BasicBlock *From, Instruction *To) { |
| SmallVector<PseudoProbeInst *, 4> ToBeMoved; |
| for (auto &I : *From) { |
| if (auto *II = dyn_cast<PseudoProbeInst>(&I)) { |
| addPseudoProbeAttribute(*II, PseudoProbeAttributes::Dangling); |
| ToBeMoved.push_back(II); |
| } |
| } |
| |
| for (auto *I : ToBeMoved) |
| I->moveBefore(To); |
| |
| return !ToBeMoved.empty(); |
| } |
| |
| /// Same dangling probes in one blocks are redundant since they all have the |
| /// same semantic that is to rely on the counts inference too to get reasonable |
| /// count for the same original block. Therefore, there's no need to keep |
| /// multiple copies of them. |
| bool removeRedundantPseudoProbes(BasicBlock *Block) { |
| |
| auto Hash = [](const PseudoProbeInst *I) { |
| return std::hash<uint64_t>()(I->getFuncGuid()->getZExtValue()) ^ |
| std::hash<uint64_t>()(I->getIndex()->getZExtValue()); |
| }; |
| |
| auto IsEqual = [](const PseudoProbeInst *Left, const PseudoProbeInst *Right) { |
| return Left->getFuncGuid() == Right->getFuncGuid() && |
| Left->getIndex() == Right->getIndex() && |
| Left->getAttributes() == Right->getAttributes() && |
| Left->getDebugLoc() == Right->getDebugLoc(); |
| }; |
| |
| SmallVector<PseudoProbeInst *, 4> ToBeRemoved; |
| std::unordered_set<PseudoProbeInst *, decltype(Hash), decltype(IsEqual)> |
| DanglingProbes(0, Hash, IsEqual); |
| |
| for (auto &I : *Block) { |
| if (auto *II = dyn_cast<PseudoProbeInst>(&I)) { |
| if (II->getAttributes()->getZExtValue() & |
| (uint32_t)PseudoProbeAttributes::Dangling) |
| if (!DanglingProbes.insert(II).second) |
| ToBeRemoved.push_back(II); |
| } |
| } |
| |
| for (auto *I : ToBeRemoved) |
| I->eraseFromParent(); |
| return !ToBeRemoved.empty(); |
| } |
| } // namespace llvm |