| //===-- RISCVExpandAtomicPseudoInsts.cpp - Expand atomic pseudo instrs. ---===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file contains a pass that expands atomic pseudo instructions into |
| // target instructions. This pass should be run at the last possible moment, |
| // avoiding the possibility for other passes to break the requirements for |
| // forward progress in the LR/SC block. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "RISCV.h" |
| #include "RISCVInstrInfo.h" |
| #include "RISCVTargetMachine.h" |
| |
| #include "llvm/CodeGen/LivePhysRegs.h" |
| #include "llvm/CodeGen/MachineFunctionPass.h" |
| #include "llvm/CodeGen/MachineInstrBuilder.h" |
| |
| using namespace llvm; |
| |
| #define RISCV_EXPAND_ATOMIC_PSEUDO_NAME \ |
| "RISCV atomic pseudo instruction expansion pass" |
| |
| namespace { |
| |
| class RISCVExpandAtomicPseudo : public MachineFunctionPass { |
| public: |
| const RISCVInstrInfo *TII; |
| static char ID; |
| |
| RISCVExpandAtomicPseudo() : MachineFunctionPass(ID) { |
| initializeRISCVExpandAtomicPseudoPass(*PassRegistry::getPassRegistry()); |
| } |
| |
| bool runOnMachineFunction(MachineFunction &MF) override; |
| |
| StringRef getPassName() const override { |
| return RISCV_EXPAND_ATOMIC_PSEUDO_NAME; |
| } |
| |
| private: |
| bool expandMBB(MachineBasicBlock &MBB); |
| bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
| MachineBasicBlock::iterator &NextMBBI); |
| bool expandAtomicBinOp(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp, |
| bool IsMasked, int Width, |
| MachineBasicBlock::iterator &NextMBBI); |
| bool expandAtomicMinMaxOp(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, |
| AtomicRMWInst::BinOp, bool IsMasked, int Width, |
| MachineBasicBlock::iterator &NextMBBI); |
| bool expandAtomicCmpXchg(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, bool IsMasked, |
| int Width, MachineBasicBlock::iterator &NextMBBI); |
| }; |
| |
| char RISCVExpandAtomicPseudo::ID = 0; |
| |
| bool RISCVExpandAtomicPseudo::runOnMachineFunction(MachineFunction &MF) { |
| TII = static_cast<const RISCVInstrInfo *>(MF.getSubtarget().getInstrInfo()); |
| bool Modified = false; |
| for (auto &MBB : MF) |
| Modified |= expandMBB(MBB); |
| return Modified; |
| } |
| |
| bool RISCVExpandAtomicPseudo::expandMBB(MachineBasicBlock &MBB) { |
| bool Modified = false; |
| |
| MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); |
| while (MBBI != E) { |
| MachineBasicBlock::iterator NMBBI = std::next(MBBI); |
| Modified |= expandMI(MBB, MBBI, NMBBI); |
| MBBI = NMBBI; |
| } |
| |
| return Modified; |
| } |
| |
| bool RISCVExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, |
| MachineBasicBlock::iterator &NextMBBI) { |
| // RISCVInstrInfo::getInstSizeInBytes hard-codes the number of expanded |
| // instructions for each pseudo, and must be updated when adding new pseudos |
| // or changing existing ones. |
| switch (MBBI->getOpcode()) { |
| case RISCV::PseudoAtomicLoadNand32: |
| return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32, |
| NextMBBI); |
| case RISCV::PseudoAtomicLoadNand64: |
| return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64, |
| NextMBBI); |
| case RISCV::PseudoMaskedAtomicSwap32: |
| return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32, |
| NextMBBI); |
| case RISCV::PseudoMaskedAtomicLoadAdd32: |
| return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI); |
| case RISCV::PseudoMaskedAtomicLoadSub32: |
| return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI); |
| case RISCV::PseudoMaskedAtomicLoadNand32: |
| return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32, |
| NextMBBI); |
| case RISCV::PseudoMaskedAtomicLoadMax32: |
| return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32, |
| NextMBBI); |
| case RISCV::PseudoMaskedAtomicLoadMin32: |
| return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32, |
| NextMBBI); |
| case RISCV::PseudoMaskedAtomicLoadUMax32: |
| return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32, |
| NextMBBI); |
| case RISCV::PseudoMaskedAtomicLoadUMin32: |
| return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32, |
| NextMBBI); |
| case RISCV::PseudoCmpXchg32: |
| return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI); |
| case RISCV::PseudoCmpXchg64: |
| return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI); |
| case RISCV::PseudoMaskedCmpXchg32: |
| return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI); |
| } |
| |
| return false; |
| } |
| |
| static unsigned getLRForRMW32(AtomicOrdering Ordering) { |
| switch (Ordering) { |
| default: |
| llvm_unreachable("Unexpected AtomicOrdering"); |
| case AtomicOrdering::Monotonic: |
| return RISCV::LR_W; |
| case AtomicOrdering::Acquire: |
| return RISCV::LR_W_AQ; |
| case AtomicOrdering::Release: |
| return RISCV::LR_W; |
| case AtomicOrdering::AcquireRelease: |
| return RISCV::LR_W_AQ; |
| case AtomicOrdering::SequentiallyConsistent: |
| return RISCV::LR_W_AQ_RL; |
| } |
| } |
| |
| static unsigned getSCForRMW32(AtomicOrdering Ordering) { |
| switch (Ordering) { |
| default: |
| llvm_unreachable("Unexpected AtomicOrdering"); |
| case AtomicOrdering::Monotonic: |
| return RISCV::SC_W; |
| case AtomicOrdering::Acquire: |
| return RISCV::SC_W; |
| case AtomicOrdering::Release: |
| return RISCV::SC_W_RL; |
| case AtomicOrdering::AcquireRelease: |
| return RISCV::SC_W_RL; |
| case AtomicOrdering::SequentiallyConsistent: |
| return RISCV::SC_W_AQ_RL; |
| } |
| } |
| |
| static unsigned getLRForRMW64(AtomicOrdering Ordering) { |
| switch (Ordering) { |
| default: |
| llvm_unreachable("Unexpected AtomicOrdering"); |
| case AtomicOrdering::Monotonic: |
| return RISCV::LR_D; |
| case AtomicOrdering::Acquire: |
| return RISCV::LR_D_AQ; |
| case AtomicOrdering::Release: |
| return RISCV::LR_D; |
| case AtomicOrdering::AcquireRelease: |
| return RISCV::LR_D_AQ; |
| case AtomicOrdering::SequentiallyConsistent: |
| return RISCV::LR_D_AQ_RL; |
| } |
| } |
| |
| static unsigned getSCForRMW64(AtomicOrdering Ordering) { |
| switch (Ordering) { |
| default: |
| llvm_unreachable("Unexpected AtomicOrdering"); |
| case AtomicOrdering::Monotonic: |
| return RISCV::SC_D; |
| case AtomicOrdering::Acquire: |
| return RISCV::SC_D; |
| case AtomicOrdering::Release: |
| return RISCV::SC_D_RL; |
| case AtomicOrdering::AcquireRelease: |
| return RISCV::SC_D_RL; |
| case AtomicOrdering::SequentiallyConsistent: |
| return RISCV::SC_D_AQ_RL; |
| } |
| } |
| |
| static unsigned getLRForRMW(AtomicOrdering Ordering, int Width) { |
| if (Width == 32) |
| return getLRForRMW32(Ordering); |
| if (Width == 64) |
| return getLRForRMW64(Ordering); |
| llvm_unreachable("Unexpected LR width\n"); |
| } |
| |
| static unsigned getSCForRMW(AtomicOrdering Ordering, int Width) { |
| if (Width == 32) |
| return getSCForRMW32(Ordering); |
| if (Width == 64) |
| return getSCForRMW64(Ordering); |
| llvm_unreachable("Unexpected SC width\n"); |
| } |
| |
| static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI, |
| DebugLoc DL, MachineBasicBlock *ThisMBB, |
| MachineBasicBlock *LoopMBB, |
| MachineBasicBlock *DoneMBB, |
| AtomicRMWInst::BinOp BinOp, int Width) { |
| Register DestReg = MI.getOperand(0).getReg(); |
| Register ScratchReg = MI.getOperand(1).getReg(); |
| Register AddrReg = MI.getOperand(2).getReg(); |
| Register IncrReg = MI.getOperand(3).getReg(); |
| AtomicOrdering Ordering = |
| static_cast<AtomicOrdering>(MI.getOperand(4).getImm()); |
| |
| // .loop: |
| // lr.[w|d] dest, (addr) |
| // binop scratch, dest, val |
| // sc.[w|d] scratch, scratch, (addr) |
| // bnez scratch, loop |
| BuildMI(LoopMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg) |
| .addReg(AddrReg); |
| switch (BinOp) { |
| default: |
| llvm_unreachable("Unexpected AtomicRMW BinOp"); |
| case AtomicRMWInst::Nand: |
| BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg) |
| .addReg(DestReg) |
| .addReg(IncrReg); |
| BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg) |
| .addReg(ScratchReg) |
| .addImm(-1); |
| break; |
| } |
| BuildMI(LoopMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg) |
| .addReg(AddrReg) |
| .addReg(ScratchReg); |
| BuildMI(LoopMBB, DL, TII->get(RISCV::BNE)) |
| .addReg(ScratchReg) |
| .addReg(RISCV::X0) |
| .addMBB(LoopMBB); |
| } |
| |
| static void insertMaskedMerge(const RISCVInstrInfo *TII, DebugLoc DL, |
| MachineBasicBlock *MBB, Register DestReg, |
| Register OldValReg, Register NewValReg, |
| Register MaskReg, Register ScratchReg) { |
| assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique"); |
| assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique"); |
| assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique"); |
| |
| // We select bits from newval and oldval using: |
| // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge |
| // r = oldval ^ ((oldval ^ newval) & masktargetdata); |
| BuildMI(MBB, DL, TII->get(RISCV::XOR), ScratchReg) |
| .addReg(OldValReg) |
| .addReg(NewValReg); |
| BuildMI(MBB, DL, TII->get(RISCV::AND), ScratchReg) |
| .addReg(ScratchReg) |
| .addReg(MaskReg); |
| BuildMI(MBB, DL, TII->get(RISCV::XOR), DestReg) |
| .addReg(OldValReg) |
| .addReg(ScratchReg); |
| } |
| |
| static void doMaskedAtomicBinOpExpansion( |
| const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL, |
| MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB, |
| MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) { |
| assert(Width == 32 && "Should never need to expand masked 64-bit operations"); |
| Register DestReg = MI.getOperand(0).getReg(); |
| Register ScratchReg = MI.getOperand(1).getReg(); |
| Register AddrReg = MI.getOperand(2).getReg(); |
| Register IncrReg = MI.getOperand(3).getReg(); |
| Register MaskReg = MI.getOperand(4).getReg(); |
| AtomicOrdering Ordering = |
| static_cast<AtomicOrdering>(MI.getOperand(5).getImm()); |
| |
| // .loop: |
| // lr.w destreg, (alignedaddr) |
| // binop scratch, destreg, incr |
| // xor scratch, destreg, scratch |
| // and scratch, scratch, masktargetdata |
| // xor scratch, destreg, scratch |
| // sc.w scratch, scratch, (alignedaddr) |
| // bnez scratch, loop |
| BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg) |
| .addReg(AddrReg); |
| switch (BinOp) { |
| default: |
| llvm_unreachable("Unexpected AtomicRMW BinOp"); |
| case AtomicRMWInst::Xchg: |
| BuildMI(LoopMBB, DL, TII->get(RISCV::ADDI), ScratchReg) |
| .addReg(IncrReg) |
| .addImm(0); |
| break; |
| case AtomicRMWInst::Add: |
| BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg) |
| .addReg(DestReg) |
| .addReg(IncrReg); |
| break; |
| case AtomicRMWInst::Sub: |
| BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg) |
| .addReg(DestReg) |
| .addReg(IncrReg); |
| break; |
| case AtomicRMWInst::Nand: |
| BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg) |
| .addReg(DestReg) |
| .addReg(IncrReg); |
| BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg) |
| .addReg(ScratchReg) |
| .addImm(-1); |
| break; |
| } |
| |
| insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg, |
| ScratchReg); |
| |
| BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg) |
| .addReg(AddrReg) |
| .addReg(ScratchReg); |
| BuildMI(LoopMBB, DL, TII->get(RISCV::BNE)) |
| .addReg(ScratchReg) |
| .addReg(RISCV::X0) |
| .addMBB(LoopMBB); |
| } |
| |
| bool RISCVExpandAtomicPseudo::expandAtomicBinOp( |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
| AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width, |
| MachineBasicBlock::iterator &NextMBBI) { |
| MachineInstr &MI = *MBBI; |
| DebugLoc DL = MI.getDebugLoc(); |
| |
| MachineFunction *MF = MBB.getParent(); |
| auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
| auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
| |
| // Insert new MBBs. |
| MF->insert(++MBB.getIterator(), LoopMBB); |
| MF->insert(++LoopMBB->getIterator(), DoneMBB); |
| |
| // Set up successors and transfer remaining instructions to DoneMBB. |
| LoopMBB->addSuccessor(LoopMBB); |
| LoopMBB->addSuccessor(DoneMBB); |
| DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end()); |
| DoneMBB->transferSuccessors(&MBB); |
| MBB.addSuccessor(LoopMBB); |
| |
| if (!IsMasked) |
| doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width); |
| else |
| doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, |
| Width); |
| |
| NextMBBI = MBB.end(); |
| MI.eraseFromParent(); |
| |
| LivePhysRegs LiveRegs; |
| computeAndAddLiveIns(LiveRegs, *LoopMBB); |
| computeAndAddLiveIns(LiveRegs, *DoneMBB); |
| |
| return true; |
| } |
| |
| static void insertSext(const RISCVInstrInfo *TII, DebugLoc DL, |
| MachineBasicBlock *MBB, Register ValReg, |
| Register ShamtReg) { |
| BuildMI(MBB, DL, TII->get(RISCV::SLL), ValReg) |
| .addReg(ValReg) |
| .addReg(ShamtReg); |
| BuildMI(MBB, DL, TII->get(RISCV::SRA), ValReg) |
| .addReg(ValReg) |
| .addReg(ShamtReg); |
| } |
| |
| bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp( |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
| AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width, |
| MachineBasicBlock::iterator &NextMBBI) { |
| assert(IsMasked == true && |
| "Should only need to expand masked atomic max/min"); |
| assert(Width == 32 && "Should never need to expand masked 64-bit operations"); |
| |
| MachineInstr &MI = *MBBI; |
| DebugLoc DL = MI.getDebugLoc(); |
| MachineFunction *MF = MBB.getParent(); |
| auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
| auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
| auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
| auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
| |
| // Insert new MBBs. |
| MF->insert(++MBB.getIterator(), LoopHeadMBB); |
| MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB); |
| MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB); |
| MF->insert(++LoopTailMBB->getIterator(), DoneMBB); |
| |
| // Set up successors and transfer remaining instructions to DoneMBB. |
| LoopHeadMBB->addSuccessor(LoopIfBodyMBB); |
| LoopHeadMBB->addSuccessor(LoopTailMBB); |
| LoopIfBodyMBB->addSuccessor(LoopTailMBB); |
| LoopTailMBB->addSuccessor(LoopHeadMBB); |
| LoopTailMBB->addSuccessor(DoneMBB); |
| DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end()); |
| DoneMBB->transferSuccessors(&MBB); |
| MBB.addSuccessor(LoopHeadMBB); |
| |
| Register DestReg = MI.getOperand(0).getReg(); |
| Register Scratch1Reg = MI.getOperand(1).getReg(); |
| Register Scratch2Reg = MI.getOperand(2).getReg(); |
| Register AddrReg = MI.getOperand(3).getReg(); |
| Register IncrReg = MI.getOperand(4).getReg(); |
| Register MaskReg = MI.getOperand(5).getReg(); |
| bool IsSigned = BinOp == AtomicRMWInst::Min || BinOp == AtomicRMWInst::Max; |
| AtomicOrdering Ordering = |
| static_cast<AtomicOrdering>(MI.getOperand(IsSigned ? 7 : 6).getImm()); |
| |
| // |
| // .loophead: |
| // lr.w destreg, (alignedaddr) |
| // and scratch2, destreg, mask |
| // mv scratch1, destreg |
| // [sext scratch2 if signed min/max] |
| // ifnochangeneeded scratch2, incr, .looptail |
| BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg) |
| .addReg(AddrReg); |
| BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), Scratch2Reg) |
| .addReg(DestReg) |
| .addReg(MaskReg); |
| BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), Scratch1Reg) |
| .addReg(DestReg) |
| .addImm(0); |
| |
| switch (BinOp) { |
| default: |
| llvm_unreachable("Unexpected AtomicRMW BinOp"); |
| case AtomicRMWInst::Max: { |
| insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg()); |
| BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE)) |
| .addReg(Scratch2Reg) |
| .addReg(IncrReg) |
| .addMBB(LoopTailMBB); |
| break; |
| } |
| case AtomicRMWInst::Min: { |
| insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg()); |
| BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE)) |
| .addReg(IncrReg) |
| .addReg(Scratch2Reg) |
| .addMBB(LoopTailMBB); |
| break; |
| } |
| case AtomicRMWInst::UMax: |
| BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU)) |
| .addReg(Scratch2Reg) |
| .addReg(IncrReg) |
| .addMBB(LoopTailMBB); |
| break; |
| case AtomicRMWInst::UMin: |
| BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU)) |
| .addReg(IncrReg) |
| .addReg(Scratch2Reg) |
| .addMBB(LoopTailMBB); |
| break; |
| } |
| |
| // .loopifbody: |
| // xor scratch1, destreg, incr |
| // and scratch1, scratch1, mask |
| // xor scratch1, destreg, scratch1 |
| insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg, |
| MaskReg, Scratch1Reg); |
| |
| // .looptail: |
| // sc.w scratch1, scratch1, (addr) |
| // bnez scratch1, loop |
| BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), Scratch1Reg) |
| .addReg(AddrReg) |
| .addReg(Scratch1Reg); |
| BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE)) |
| .addReg(Scratch1Reg) |
| .addReg(RISCV::X0) |
| .addMBB(LoopHeadMBB); |
| |
| NextMBBI = MBB.end(); |
| MI.eraseFromParent(); |
| |
| LivePhysRegs LiveRegs; |
| computeAndAddLiveIns(LiveRegs, *LoopHeadMBB); |
| computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB); |
| computeAndAddLiveIns(LiveRegs, *LoopTailMBB); |
| computeAndAddLiveIns(LiveRegs, *DoneMBB); |
| |
| return true; |
| } |
| |
| bool RISCVExpandAtomicPseudo::expandAtomicCmpXchg( |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked, |
| int Width, MachineBasicBlock::iterator &NextMBBI) { |
| MachineInstr &MI = *MBBI; |
| DebugLoc DL = MI.getDebugLoc(); |
| MachineFunction *MF = MBB.getParent(); |
| auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
| auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
| auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
| |
| // Insert new MBBs. |
| MF->insert(++MBB.getIterator(), LoopHeadMBB); |
| MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB); |
| MF->insert(++LoopTailMBB->getIterator(), DoneMBB); |
| |
| // Set up successors and transfer remaining instructions to DoneMBB. |
| LoopHeadMBB->addSuccessor(LoopTailMBB); |
| LoopHeadMBB->addSuccessor(DoneMBB); |
| LoopTailMBB->addSuccessor(DoneMBB); |
| LoopTailMBB->addSuccessor(LoopHeadMBB); |
| DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end()); |
| DoneMBB->transferSuccessors(&MBB); |
| MBB.addSuccessor(LoopHeadMBB); |
| |
| Register DestReg = MI.getOperand(0).getReg(); |
| Register ScratchReg = MI.getOperand(1).getReg(); |
| Register AddrReg = MI.getOperand(2).getReg(); |
| Register CmpValReg = MI.getOperand(3).getReg(); |
| Register NewValReg = MI.getOperand(4).getReg(); |
| AtomicOrdering Ordering = |
| static_cast<AtomicOrdering>(MI.getOperand(IsMasked ? 6 : 5).getImm()); |
| |
| if (!IsMasked) { |
| // .loophead: |
| // lr.[w|d] dest, (addr) |
| // bne dest, cmpval, done |
| BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg) |
| .addReg(AddrReg); |
| BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE)) |
| .addReg(DestReg) |
| .addReg(CmpValReg) |
| .addMBB(DoneMBB); |
| // .looptail: |
| // sc.[w|d] scratch, newval, (addr) |
| // bnez scratch, loophead |
| BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg) |
| .addReg(AddrReg) |
| .addReg(NewValReg); |
| BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE)) |
| .addReg(ScratchReg) |
| .addReg(RISCV::X0) |
| .addMBB(LoopHeadMBB); |
| } else { |
| // .loophead: |
| // lr.w dest, (addr) |
| // and scratch, dest, mask |
| // bne scratch, cmpval, done |
| Register MaskReg = MI.getOperand(5).getReg(); |
| BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg) |
| .addReg(AddrReg); |
| BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), ScratchReg) |
| .addReg(DestReg) |
| .addReg(MaskReg); |
| BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE)) |
| .addReg(ScratchReg) |
| .addReg(CmpValReg) |
| .addMBB(DoneMBB); |
| |
| // .looptail: |
| // xor scratch, dest, newval |
| // and scratch, scratch, mask |
| // xor scratch, dest, scratch |
| // sc.w scratch, scratch, (adrr) |
| // bnez scratch, loophead |
| insertMaskedMerge(TII, DL, LoopTailMBB, ScratchReg, DestReg, NewValReg, |
| MaskReg, ScratchReg); |
| BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg) |
| .addReg(AddrReg) |
| .addReg(ScratchReg); |
| BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE)) |
| .addReg(ScratchReg) |
| .addReg(RISCV::X0) |
| .addMBB(LoopHeadMBB); |
| } |
| |
| NextMBBI = MBB.end(); |
| MI.eraseFromParent(); |
| |
| LivePhysRegs LiveRegs; |
| computeAndAddLiveIns(LiveRegs, *LoopHeadMBB); |
| computeAndAddLiveIns(LiveRegs, *LoopTailMBB); |
| computeAndAddLiveIns(LiveRegs, *DoneMBB); |
| |
| return true; |
| } |
| |
| } // end of anonymous namespace |
| |
| INITIALIZE_PASS(RISCVExpandAtomicPseudo, "riscv-expand-atomic-pseudo", |
| RISCV_EXPAND_ATOMIC_PSEUDO_NAME, false, false) |
| |
| namespace llvm { |
| |
| FunctionPass *createRISCVExpandAtomicPseudoPass() { |
| return new RISCVExpandAtomicPseudo(); |
| } |
| |
| } // end of namespace llvm |