blob: 19f824850607ca377b6b7797e9941d1249268e95 [file] [log] [blame]
//===-- BasicBlockPathCloning.cpp ---=========-----------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
/// \file
/// BasicBlockPathCloning implementation.
///
/// The purpose of this pass is to clone basic block paths based on information
/// provided by the -fbasic-block-sections=list option.
/// Please refer to BasicBlockSectionsProfileReader.cpp to see a path cloning
/// example.
//===----------------------------------------------------------------------===//
// This pass clones the machine basic blocks alongs the given paths and sets up
// the CFG. It assigns BBIDs to the cloned blocks so that the
// `BasicBlockSections` pass can correctly map the cluster information to the
// blocks. The cloned block's BBID will have the same BaseID as the original
// block, but will get a unique non-zero CloneID (original blocks all have zero
// CloneIDs). This pass applies a path cloning if it satisfies the following
// conditions:
// 1. All BBIDs in the path should be mapped to existing blocks.
// 2. Each two consecutive BBIDs in the path must have a successor
// relationship in the CFG.
// 3. The path should not include a block with indirect branches, except for
// the last block.
// If a path does not satisfy all three conditions, it will be rejected, but the
// CloneIDs for its (supposed to be cloned) blocks will be bypassed to make sure
// that the `BasicBlockSections` pass can map cluster info correctly to the
// actually-cloned blocks.
//===----------------------------------------------------------------------===//
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/CodeGen/BasicBlockSectionUtils.h"
#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/InitializePasses.h"
#include "llvm/Support/WithColor.h"
#include "llvm/Target/TargetMachine.h"
using namespace llvm;
namespace {
// Clones the given block and assigns the given `CloneID` to its BBID. Copies
// the instructions into the new block and sets up its successors.
MachineBasicBlock *CloneMachineBasicBlock(MachineBasicBlock &OrigBB,
unsigned CloneID) {
auto &MF = *OrigBB.getParent();
auto TII = MF.getSubtarget().getInstrInfo();
// Create the clone block and set its BBID based on the original block.
MachineBasicBlock *CloneBB = MF.CreateMachineBasicBlock(
OrigBB.getBasicBlock(), UniqueBBID{OrigBB.getBBID()->BaseID, CloneID});
MF.push_back(CloneBB);
// Copy the instructions.
for (auto &I : OrigBB.instrs()) {
// Bundled instructions are duplicated together.
if (I.isBundledWithPred())
continue;
TII->duplicate(*CloneBB, CloneBB->end(), I);
}
// Add the successors of the original block as the new block's successors.
// We set the predecessor after returning from this call.
for (auto SI = OrigBB.succ_begin(), SE = OrigBB.succ_end(); SI != SE; ++SI)
CloneBB->copySuccessor(&OrigBB, SI);
if (auto FT = OrigBB.getFallThrough(/*JumpToFallThrough=*/false)) {
// The original block has an implicit fall through.
// Insert an explicit unconditional jump from the cloned block to the
// fallthrough block. Technically, this is only needed for the last block
// of the path, but we do it for all clones for consistency.
TII->insertUnconditionalBranch(*CloneBB, FT, CloneBB->findBranchDebugLoc());
}
return CloneBB;
}
// Returns if we can legally apply the cloning represented by `ClonePath`.
// `BBIDToBlock` contains the original basic blocks in function `MF` keyed by
// their `BBID::BaseID`.
bool IsValidCloning(const MachineFunction &MF,
const DenseMap<unsigned, MachineBasicBlock *> &BBIDToBlock,
const SmallVector<unsigned> &ClonePath) {
const MachineBasicBlock *PrevBB = nullptr;
for (size_t I = 0; I < ClonePath.size(); ++I) {
unsigned BBID = ClonePath[I];
const MachineBasicBlock *PathBB = BBIDToBlock.lookup(BBID);
if (!PathBB) {
WithColor::warning() << "no block with id " << BBID << " in function "
<< MF.getName() << "\n";
return false;
}
if (PrevBB) {
if (!PrevBB->isSuccessor(PathBB)) {
WithColor::warning()
<< "block #" << BBID << " is not a successor of block #"
<< PrevBB->getBBID()->BaseID << " in function " << MF.getName()
<< "\n";
return false;
}
for (auto &MI : *PathBB) {
// Avoid cloning when the block contains non-duplicable instructions.
// CFI instructions are marked as non-duplicable only because of Darwin,
// so we exclude them from this check.
if (MI.isNotDuplicable() && !MI.isCFIInstruction()) {
WithColor::warning()
<< "block #" << BBID
<< " has non-duplicable instructions in function " << MF.getName()
<< "\n";
return false;
}
}
if (PathBB->isMachineBlockAddressTaken()) {
// Avoid cloning blocks which have their address taken since we can't
// rewire branches to those blocks as easily (e.g., branches within
// inline assembly).
WithColor::warning()
<< "block #" << BBID
<< " has its machine block address taken in function "
<< MF.getName() << "\n";
return false;
}
}
if (I != ClonePath.size() - 1 && !PathBB->empty() &&
PathBB->back().isIndirectBranch()) {
WithColor::warning()
<< "block #" << BBID
<< " has indirect branch and appears as the non-tail block of a "
"path in function "
<< MF.getName() << "\n";
return false;
}
PrevBB = PathBB;
}
return true;
}
// Applies all clonings specified in `ClonePaths` to `MF`. Returns true
// if any clonings have been applied.
bool ApplyCloning(MachineFunction &MF,
const SmallVector<SmallVector<unsigned>> &ClonePaths) {
if (ClonePaths.empty())
return false;
bool AnyPathsCloned = false;
// Map from the final BB IDs to the `MachineBasicBlock`s.
DenseMap<unsigned, MachineBasicBlock *> BBIDToBlock;
for (auto &BB : MF)
BBIDToBlock.try_emplace(BB.getBBID()->BaseID, &BB);
DenseMap<unsigned, unsigned> NClonesForBBID;
auto TII = MF.getSubtarget().getInstrInfo();
for (const auto &ClonePath : ClonePaths) {
if (!IsValidCloning(MF, BBIDToBlock, ClonePath)) {
// We still need to increment the number of clones so we can map
// to the cluster info correctly.
for (unsigned BBID : ClonePath)
++NClonesForBBID[BBID];
continue;
}
MachineBasicBlock *PrevBB = nullptr;
for (unsigned BBID : ClonePath) {
MachineBasicBlock *OrigBB = BBIDToBlock.at(BBID);
if (PrevBB == nullptr) {
// The first block in the path is not cloned. We only need to make it
// branch to the next cloned block in the path. Here, we make its
// fallthrough explicit so we can change it later.
if (auto FT = OrigBB->getFallThrough(/*JumpToFallThrough=*/false)) {
TII->insertUnconditionalBranch(*OrigBB, FT,
OrigBB->findBranchDebugLoc());
}
PrevBB = OrigBB;
continue;
}
MachineBasicBlock *CloneBB =
CloneMachineBasicBlock(*OrigBB, ++NClonesForBBID[BBID]);
// Set up the previous block in the path to jump to the clone. This also
// transfers the successor/predecessor relationship of PrevBB and OrigBB
// to that of PrevBB and CloneBB.
PrevBB->ReplaceUsesOfBlockWith(OrigBB, CloneBB);
// Copy the livein set.
for (auto &LiveIn : OrigBB->liveins())
CloneBB->addLiveIn(LiveIn);
PrevBB = CloneBB;
}
AnyPathsCloned = true;
}
return AnyPathsCloned;
}
} // end anonymous namespace
namespace llvm {
class BasicBlockPathCloning : public MachineFunctionPass {
public:
static char ID;
BasicBlockSectionsProfileReaderWrapperPass *BBSectionsProfileReader = nullptr;
BasicBlockPathCloning() : MachineFunctionPass(ID) {
initializeBasicBlockPathCloningPass(*PassRegistry::getPassRegistry());
}
StringRef getPassName() const override { return "Basic Block Path Cloning"; }
void getAnalysisUsage(AnalysisUsage &AU) const override;
/// Identify basic blocks that need separate sections and prepare to emit them
/// accordingly.
bool runOnMachineFunction(MachineFunction &MF) override;
};
} // namespace llvm
char BasicBlockPathCloning::ID = 0;
INITIALIZE_PASS_BEGIN(
BasicBlockPathCloning, "bb-path-cloning",
"Applies path clonings for the -basic-block-sections=list option", false,
false)
INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReaderWrapperPass)
INITIALIZE_PASS_END(
BasicBlockPathCloning, "bb-path-cloning",
"Applies path clonings for the -basic-block-sections=list option", false,
false)
bool BasicBlockPathCloning::runOnMachineFunction(MachineFunction &MF) {
assert(MF.getTarget().getBBSectionsType() == BasicBlockSection::List &&
"BB Sections list not enabled!");
if (hasInstrProfHashMismatch(MF))
return false;
return ApplyCloning(MF,
getAnalysis<BasicBlockSectionsProfileReaderWrapperPass>()
.getClonePathsForFunction(MF.getName()));
}
void BasicBlockPathCloning::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesAll();
AU.addRequired<BasicBlockSectionsProfileReaderWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
}
MachineFunctionPass *llvm::createBasicBlockPathCloningPass() {
return new BasicBlockPathCloning();
}