| //=- LoongArchISelLowering.cpp - LoongArch DAG Lowering Implementation ---===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file defines the interfaces that LoongArch uses to lower LLVM code into |
| // a selection DAG. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "LoongArchISelLowering.h" |
| #include "LoongArch.h" |
| #include "LoongArchMachineFunctionInfo.h" |
| #include "LoongArchRegisterInfo.h" |
| #include "LoongArchSubtarget.h" |
| #include "LoongArchTargetMachine.h" |
| #include "MCTargetDesc/LoongArchBaseInfo.h" |
| #include "MCTargetDesc/LoongArchMCTargetDesc.h" |
| #include "llvm/ADT/Statistic.h" |
| #include "llvm/CodeGen/ISDOpcodes.h" |
| #include "llvm/CodeGen/RuntimeLibcalls.h" |
| #include "llvm/IR/IRBuilder.h" |
| #include "llvm/IR/IntrinsicsLoongArch.h" |
| #include "llvm/Support/Debug.h" |
| #include "llvm/Support/KnownBits.h" |
| #include "llvm/Support/MathExtras.h" |
| |
| using namespace llvm; |
| |
| #define DEBUG_TYPE "loongarch-isel-lowering" |
| |
| STATISTIC(NumTailCalls, "Number of tail calls"); |
| |
| static cl::opt<bool> ZeroDivCheck( |
| "loongarch-check-zero-division", cl::Hidden, |
| cl::desc("Trap on integer division by zero."), |
| cl::init(false)); |
| |
| LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM, |
| const LoongArchSubtarget &STI) |
| : TargetLowering(TM), Subtarget(STI) { |
| |
| MVT GRLenVT = Subtarget.getGRLenVT(); |
| // Set up the register classes. |
| addRegisterClass(GRLenVT, &LoongArch::GPRRegClass); |
| if (Subtarget.hasBasicF()) |
| addRegisterClass(MVT::f32, &LoongArch::FPR32RegClass); |
| if (Subtarget.hasBasicD()) |
| addRegisterClass(MVT::f64, &LoongArch::FPR64RegClass); |
| |
| setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, GRLenVT, |
| MVT::i1, Promote); |
| |
| // TODO: add necessary setOperationAction calls later. |
| setOperationAction(ISD::SHL_PARTS, GRLenVT, Custom); |
| setOperationAction(ISD::SRA_PARTS, GRLenVT, Custom); |
| setOperationAction(ISD::SRL_PARTS, GRLenVT, Custom); |
| setOperationAction(ISD::FP_TO_SINT, GRLenVT, Custom); |
| setOperationAction(ISD::ROTL, GRLenVT, Expand); |
| setOperationAction(ISD::CTPOP, GRLenVT, Expand); |
| setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); |
| setOperationAction(ISD::TRAP, MVT::Other, Legal); |
| setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); |
| setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); |
| |
| setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool, |
| ISD::JumpTable}, |
| GRLenVT, Custom); |
| |
| setOperationAction(ISD::GlobalTLSAddress, GRLenVT, Custom); |
| |
| setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); |
| |
| setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom); |
| if (Subtarget.is64Bit()) |
| setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom); |
| |
| setOperationAction(ISD::DYNAMIC_STACKALLOC, GRLenVT, Expand); |
| setOperationAction({ISD::STACKSAVE, ISD::STACKRESTORE}, MVT::Other, Expand); |
| setOperationAction(ISD::VASTART, MVT::Other, Custom); |
| setOperationAction({ISD::VAARG, ISD::VACOPY, ISD::VAEND}, MVT::Other, Expand); |
| |
| if (Subtarget.is64Bit()) { |
| setOperationAction(ISD::SHL, MVT::i32, Custom); |
| setOperationAction(ISD::SRA, MVT::i32, Custom); |
| setOperationAction(ISD::SRL, MVT::i32, Custom); |
| setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); |
| setOperationAction(ISD::BITCAST, MVT::i32, Custom); |
| setOperationAction(ISD::ROTR, MVT::i32, Custom); |
| setOperationAction(ISD::ROTL, MVT::i32, Custom); |
| setOperationAction(ISD::CTTZ, MVT::i32, Custom); |
| setOperationAction(ISD::CTLZ, MVT::i32, Custom); |
| setOperationAction(ISD::INTRINSIC_VOID, MVT::i32, Custom); |
| setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i32, Custom); |
| setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); |
| setOperationAction(ISD::READ_REGISTER, MVT::i32, Custom); |
| setOperationAction(ISD::WRITE_REGISTER, MVT::i32, Custom); |
| if (Subtarget.hasBasicF() && !Subtarget.hasBasicD()) |
| setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); |
| if (Subtarget.hasBasicF()) |
| setOperationAction(ISD::FRINT, MVT::f32, Legal); |
| if (Subtarget.hasBasicD()) |
| setOperationAction(ISD::FRINT, MVT::f64, Legal); |
| } |
| |
| // LA32 does not have REVB.2W and REVB.D due to the 64-bit operands, and |
| // the narrower REVB.W does not exist. But LA32 does have REVB.2H, so i16 |
| // and i32 could still be byte-swapped relatively cheaply. |
| setOperationAction(ISD::BSWAP, MVT::i16, Custom); |
| if (Subtarget.is64Bit()) { |
| setOperationAction(ISD::BSWAP, MVT::i32, Custom); |
| } |
| |
| // Expand bitreverse.i16 with native-width bitrev and shift for now, before |
| // we get to know which of sll and revb.2h is faster. |
| setOperationAction(ISD::BITREVERSE, MVT::i8, Custom); |
| if (Subtarget.is64Bit()) { |
| setOperationAction(ISD::BITREVERSE, MVT::i32, Custom); |
| setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); |
| } else { |
| setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); |
| setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom); |
| setOperationAction(ISD::READ_REGISTER, MVT::i64, Custom); |
| setOperationAction(ISD::WRITE_REGISTER, MVT::i64, Custom); |
| setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); |
| setOperationAction(ISD::INTRINSIC_VOID, MVT::i64, Custom); |
| } |
| |
| static const ISD::CondCode FPCCToExpand[] = { |
| ISD::SETOGT, ISD::SETOGE, ISD::SETUGT, ISD::SETUGE, |
| ISD::SETGE, ISD::SETNE, ISD::SETGT}; |
| |
| if (Subtarget.hasBasicF()) { |
| setCondCodeAction(FPCCToExpand, MVT::f32, Expand); |
| setOperationAction(ISD::SELECT_CC, MVT::f32, Expand); |
| setOperationAction(ISD::BR_CC, MVT::f32, Expand); |
| setOperationAction(ISD::FMA, MVT::f32, Legal); |
| setOperationAction(ISD::FMINNUM_IEEE, MVT::f32, Legal); |
| setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal); |
| setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal); |
| setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Legal); |
| setOperationAction(ISD::FSIN, MVT::f32, Expand); |
| setOperationAction(ISD::FCOS, MVT::f32, Expand); |
| setOperationAction(ISD::FSINCOS, MVT::f32, Expand); |
| setOperationAction(ISD::FPOW, MVT::f32, Expand); |
| setOperationAction(ISD::FREM, MVT::f32, Expand); |
| } |
| if (Subtarget.hasBasicD()) { |
| setCondCodeAction(FPCCToExpand, MVT::f64, Expand); |
| setOperationAction(ISD::SELECT_CC, MVT::f64, Expand); |
| setOperationAction(ISD::BR_CC, MVT::f64, Expand); |
| setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal); |
| setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Legal); |
| setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); |
| setOperationAction(ISD::FMA, MVT::f64, Legal); |
| setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal); |
| setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal); |
| setOperationAction(ISD::FSIN, MVT::f64, Expand); |
| setOperationAction(ISD::FCOS, MVT::f64, Expand); |
| setOperationAction(ISD::FSINCOS, MVT::f64, Expand); |
| setOperationAction(ISD::FPOW, MVT::f64, Expand); |
| setOperationAction(ISD::FREM, MVT::f64, Expand); |
| setTruncStoreAction(MVT::f64, MVT::f32, Expand); |
| } |
| |
| setOperationAction(ISD::BR_JT, MVT::Other, Expand); |
| |
| setOperationAction(ISD::BR_CC, GRLenVT, Expand); |
| setOperationAction(ISD::SELECT_CC, GRLenVT, Expand); |
| setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); |
| setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, GRLenVT, Expand); |
| if (!Subtarget.is64Bit()) |
| setLibcallName(RTLIB::MUL_I128, nullptr); |
| |
| setOperationAction(ISD::FP_TO_UINT, GRLenVT, Custom); |
| setOperationAction(ISD::UINT_TO_FP, GRLenVT, Expand); |
| if ((Subtarget.is64Bit() && Subtarget.hasBasicF() && |
| !Subtarget.hasBasicD())) { |
| setOperationAction(ISD::SINT_TO_FP, GRLenVT, Custom); |
| setOperationAction(ISD::UINT_TO_FP, GRLenVT, Custom); |
| } |
| |
| // Compute derived properties from the register classes. |
| computeRegisterProperties(STI.getRegisterInfo()); |
| |
| setStackPointerRegisterToSaveRestore(LoongArch::R3); |
| |
| setBooleanContents(ZeroOrOneBooleanContent); |
| |
| setMaxAtomicSizeInBitsSupported(Subtarget.getGRLen()); |
| |
| setMinCmpXchgSizeInBits(32); |
| |
| // Function alignments. |
| const Align FunctionAlignment(4); |
| setMinFunctionAlignment(FunctionAlignment); |
| |
| setTargetDAGCombine(ISD::AND); |
| setTargetDAGCombine(ISD::OR); |
| setTargetDAGCombine(ISD::SRL); |
| } |
| |
| bool LoongArchTargetLowering::isOffsetFoldingLegal( |
| const GlobalAddressSDNode *GA) const { |
| // In order to maximise the opportunity for common subexpression elimination, |
| // keep a separate ADD node for the global address offset instead of folding |
| // it in the global address node. Later peephole optimisations may choose to |
| // fold it back in when profitable. |
| return false; |
| } |
| |
| SDValue LoongArchTargetLowering::LowerOperation(SDValue Op, |
| SelectionDAG &DAG) const { |
| switch (Op.getOpcode()) { |
| case ISD::EH_DWARF_CFA: |
| return lowerEH_DWARF_CFA(Op, DAG); |
| case ISD::GlobalAddress: |
| return lowerGlobalAddress(Op, DAG); |
| case ISD::GlobalTLSAddress: |
| return lowerGlobalTLSAddress(Op, DAG); |
| case ISD::INTRINSIC_WO_CHAIN: |
| return lowerINTRINSIC_WO_CHAIN(Op, DAG); |
| case ISD::INTRINSIC_W_CHAIN: |
| return lowerINTRINSIC_W_CHAIN(Op, DAG); |
| case ISD::INTRINSIC_VOID: |
| return lowerINTRINSIC_VOID(Op, DAG); |
| case ISD::BlockAddress: |
| return lowerBlockAddress(Op, DAG); |
| case ISD::JumpTable: |
| return lowerJumpTable(Op, DAG); |
| case ISD::SHL_PARTS: |
| return lowerShiftLeftParts(Op, DAG); |
| case ISD::SRA_PARTS: |
| return lowerShiftRightParts(Op, DAG, true); |
| case ISD::SRL_PARTS: |
| return lowerShiftRightParts(Op, DAG, false); |
| case ISD::ConstantPool: |
| return lowerConstantPool(Op, DAG); |
| case ISD::FP_TO_SINT: |
| return lowerFP_TO_SINT(Op, DAG); |
| case ISD::BITCAST: |
| return lowerBITCAST(Op, DAG); |
| case ISD::UINT_TO_FP: |
| return lowerUINT_TO_FP(Op, DAG); |
| case ISD::SINT_TO_FP: |
| return lowerSINT_TO_FP(Op, DAG); |
| case ISD::VASTART: |
| return lowerVASTART(Op, DAG); |
| case ISD::FRAMEADDR: |
| return lowerFRAMEADDR(Op, DAG); |
| case ISD::RETURNADDR: |
| return lowerRETURNADDR(Op, DAG); |
| case ISD::WRITE_REGISTER: |
| return lowerWRITE_REGISTER(Op, DAG); |
| } |
| return SDValue(); |
| } |
| |
| SDValue LoongArchTargetLowering::lowerWRITE_REGISTER(SDValue Op, |
| SelectionDAG &DAG) const { |
| |
| if (Subtarget.is64Bit() && Op.getOperand(2).getValueType() == MVT::i32) { |
| DAG.getContext()->emitError( |
| "On LA64, only 64-bit registers can be written."); |
| return Op.getOperand(0); |
| } |
| |
| if (!Subtarget.is64Bit() && Op.getOperand(2).getValueType() == MVT::i64) { |
| DAG.getContext()->emitError( |
| "On LA32, only 32-bit registers can be written."); |
| return Op.getOperand(0); |
| } |
| |
| return Op; |
| } |
| |
| SDValue LoongArchTargetLowering::lowerFRAMEADDR(SDValue Op, |
| SelectionDAG &DAG) const { |
| if (!isa<ConstantSDNode>(Op.getOperand(0))) { |
| DAG.getContext()->emitError("argument to '__builtin_frame_address' must " |
| "be a constant integer"); |
| return SDValue(); |
| } |
| |
| MachineFunction &MF = DAG.getMachineFunction(); |
| MF.getFrameInfo().setFrameAddressIsTaken(true); |
| Register FrameReg = Subtarget.getRegisterInfo()->getFrameRegister(MF); |
| EVT VT = Op.getValueType(); |
| SDLoc DL(Op); |
| SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT); |
| unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); |
| int GRLenInBytes = Subtarget.getGRLen() / 8; |
| |
| while (Depth--) { |
| int Offset = -(GRLenInBytes * 2); |
| SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr, |
| DAG.getIntPtrConstant(Offset, DL)); |
| FrameAddr = |
| DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo()); |
| } |
| return FrameAddr; |
| } |
| |
| SDValue LoongArchTargetLowering::lowerRETURNADDR(SDValue Op, |
| SelectionDAG &DAG) const { |
| if (verifyReturnAddressArgumentIsConstant(Op, DAG)) |
| return SDValue(); |
| |
| // Currently only support lowering return address for current frame. |
| if (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue() != 0) { |
| DAG.getContext()->emitError( |
| "return address can only be determined for the current frame"); |
| return SDValue(); |
| } |
| |
| MachineFunction &MF = DAG.getMachineFunction(); |
| MF.getFrameInfo().setReturnAddressIsTaken(true); |
| MVT GRLenVT = Subtarget.getGRLenVT(); |
| |
| // Return the value of the return address register, marking it an implicit |
| // live-in. |
| Register Reg = MF.addLiveIn(Subtarget.getRegisterInfo()->getRARegister(), |
| getRegClassFor(GRLenVT)); |
| return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), Reg, GRLenVT); |
| } |
| |
| SDValue LoongArchTargetLowering::lowerEH_DWARF_CFA(SDValue Op, |
| SelectionDAG &DAG) const { |
| MachineFunction &MF = DAG.getMachineFunction(); |
| auto Size = Subtarget.getGRLen() / 8; |
| auto FI = MF.getFrameInfo().CreateFixedObject(Size, 0, false); |
| return DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout())); |
| } |
| |
| SDValue LoongArchTargetLowering::lowerVASTART(SDValue Op, |
| SelectionDAG &DAG) const { |
| MachineFunction &MF = DAG.getMachineFunction(); |
| auto *FuncInfo = MF.getInfo<LoongArchMachineFunctionInfo>(); |
| |
| SDLoc DL(Op); |
| SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), |
| getPointerTy(MF.getDataLayout())); |
| |
| // vastart just stores the address of the VarArgsFrameIndex slot into the |
| // memory location argument. |
| const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); |
| return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1), |
| MachinePointerInfo(SV)); |
| } |
| |
| SDValue LoongArchTargetLowering::lowerUINT_TO_FP(SDValue Op, |
| SelectionDAG &DAG) const { |
| assert(Subtarget.is64Bit() && Subtarget.hasBasicF() && |
| !Subtarget.hasBasicD() && "unexpected target features"); |
| |
| SDLoc DL(Op); |
| SDValue Op0 = Op.getOperand(0); |
| if (Op0->getOpcode() == ISD::AND) { |
| auto *C = dyn_cast<ConstantSDNode>(Op0.getOperand(1)); |
| if (C && C->getZExtValue() < UINT64_C(0xFFFFFFFF)) |
| return Op; |
| } |
| |
| if (Op0->getOpcode() == LoongArchISD::BSTRPICK && |
| Op0.getConstantOperandVal(1) < UINT64_C(0X1F) && |
| Op0.getConstantOperandVal(2) == UINT64_C(0)) |
| return Op; |
| |
| if (Op0.getOpcode() == ISD::AssertZext && |
| dyn_cast<VTSDNode>(Op0.getOperand(1))->getVT().bitsLT(MVT::i32)) |
| return Op; |
| |
| EVT OpVT = Op0.getValueType(); |
| EVT RetVT = Op.getValueType(); |
| RTLIB::Libcall LC = RTLIB::getUINTTOFP(OpVT, RetVT); |
| MakeLibCallOptions CallOptions; |
| CallOptions.setTypeListBeforeSoften(OpVT, RetVT, true); |
| SDValue Chain = SDValue(); |
| SDValue Result; |
| std::tie(Result, Chain) = |
| makeLibCall(DAG, LC, Op.getValueType(), Op0, CallOptions, DL, Chain); |
| return Result; |
| } |
| |
| SDValue LoongArchTargetLowering::lowerSINT_TO_FP(SDValue Op, |
| SelectionDAG &DAG) const { |
| assert(Subtarget.is64Bit() && Subtarget.hasBasicF() && |
| !Subtarget.hasBasicD() && "unexpected target features"); |
| |
| SDLoc DL(Op); |
| SDValue Op0 = Op.getOperand(0); |
| |
| if ((Op0.getOpcode() == ISD::AssertSext || |
| Op0.getOpcode() == ISD::SIGN_EXTEND_INREG) && |
| dyn_cast<VTSDNode>(Op0.getOperand(1))->getVT().bitsLE(MVT::i32)) |
| return Op; |
| |
| EVT OpVT = Op0.getValueType(); |
| EVT RetVT = Op.getValueType(); |
| RTLIB::Libcall LC = RTLIB::getSINTTOFP(OpVT, RetVT); |
| MakeLibCallOptions CallOptions; |
| CallOptions.setTypeListBeforeSoften(OpVT, RetVT, true); |
| SDValue Chain = SDValue(); |
| SDValue Result; |
| std::tie(Result, Chain) = |
| makeLibCall(DAG, LC, Op.getValueType(), Op0, CallOptions, DL, Chain); |
| return Result; |
| } |
| |
| SDValue LoongArchTargetLowering::lowerBITCAST(SDValue Op, |
| SelectionDAG &DAG) const { |
| |
| SDLoc DL(Op); |
| SDValue Op0 = Op.getOperand(0); |
| |
| if (Op.getValueType() == MVT::f32 && Op0.getValueType() == MVT::i32 && |
| Subtarget.is64Bit() && Subtarget.hasBasicF()) { |
| SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0); |
| return DAG.getNode(LoongArchISD::MOVGR2FR_W_LA64, DL, MVT::f32, NewOp0); |
| } |
| return Op; |
| } |
| |
| SDValue LoongArchTargetLowering::lowerFP_TO_SINT(SDValue Op, |
| SelectionDAG &DAG) const { |
| |
| SDLoc DL(Op); |
| |
| if (Op.getValueSizeInBits() > 32 && Subtarget.hasBasicF() && |
| !Subtarget.hasBasicD()) { |
| SDValue Dst = |
| DAG.getNode(LoongArchISD::FTINT, DL, MVT::f32, Op.getOperand(0)); |
| return DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, MVT::i64, Dst); |
| } |
| |
| EVT FPTy = EVT::getFloatingPointVT(Op.getValueSizeInBits()); |
| SDValue Trunc = DAG.getNode(LoongArchISD::FTINT, DL, FPTy, Op.getOperand(0)); |
| return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Trunc); |
| } |
| |
| static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty, |
| SelectionDAG &DAG, unsigned Flags) { |
| return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags); |
| } |
| |
| static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty, |
| SelectionDAG &DAG, unsigned Flags) { |
| return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(), |
| Flags); |
| } |
| |
| static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty, |
| SelectionDAG &DAG, unsigned Flags) { |
| return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(), |
| N->getOffset(), Flags); |
| } |
| |
| static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty, |
| SelectionDAG &DAG, unsigned Flags) { |
| return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags); |
| } |
| |
| template <class NodeTy> |
| SDValue LoongArchTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG, |
| bool IsLocal) const { |
| SDLoc DL(N); |
| EVT Ty = getPointerTy(DAG.getDataLayout()); |
| SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0); |
| // TODO: Check CodeModel. |
| if (IsLocal) |
| // This generates the pattern (PseudoLA_PCREL sym), which expands to |
| // (addi.w/d (pcalau12i %pc_hi20(sym)) %pc_lo12(sym)). |
| return SDValue(DAG.getMachineNode(LoongArch::PseudoLA_PCREL, DL, Ty, Addr), |
| 0); |
| |
| // This generates the pattern (PseudoLA_GOT sym), which expands to (ld.w/d |
| // (pcalau12i %got_pc_hi20(sym)) %got_pc_lo12(sym)). |
| return SDValue(DAG.getMachineNode(LoongArch::PseudoLA_GOT, DL, Ty, Addr), 0); |
| } |
| |
| SDValue LoongArchTargetLowering::lowerBlockAddress(SDValue Op, |
| SelectionDAG &DAG) const { |
| return getAddr(cast<BlockAddressSDNode>(Op), DAG); |
| } |
| |
| SDValue LoongArchTargetLowering::lowerJumpTable(SDValue Op, |
| SelectionDAG &DAG) const { |
| return getAddr(cast<JumpTableSDNode>(Op), DAG); |
| } |
| |
| SDValue LoongArchTargetLowering::lowerConstantPool(SDValue Op, |
| SelectionDAG &DAG) const { |
| return getAddr(cast<ConstantPoolSDNode>(Op), DAG); |
| } |
| |
| SDValue LoongArchTargetLowering::lowerGlobalAddress(SDValue Op, |
| SelectionDAG &DAG) const { |
| GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op); |
| assert(N->getOffset() == 0 && "unexpected offset in global node"); |
| return getAddr(N, DAG, N->getGlobal()->isDSOLocal()); |
| } |
| |
| SDValue LoongArchTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N, |
| SelectionDAG &DAG, |
| unsigned Opc) const { |
| SDLoc DL(N); |
| EVT Ty = getPointerTy(DAG.getDataLayout()); |
| MVT GRLenVT = Subtarget.getGRLenVT(); |
| |
| SDValue Addr = DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, 0); |
| SDValue Offset = SDValue(DAG.getMachineNode(Opc, DL, Ty, Addr), 0); |
| |
| // Add the thread pointer. |
| return DAG.getNode(ISD::ADD, DL, Ty, Offset, |
| DAG.getRegister(LoongArch::R2, GRLenVT)); |
| } |
| |
| SDValue LoongArchTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N, |
| SelectionDAG &DAG, |
| unsigned Opc) const { |
| SDLoc DL(N); |
| EVT Ty = getPointerTy(DAG.getDataLayout()); |
| IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits()); |
| |
| // Use a PC-relative addressing mode to access the dynamic GOT address. |
| SDValue Addr = DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, 0); |
| SDValue Load = SDValue(DAG.getMachineNode(Opc, DL, Ty, Addr), 0); |
| |
| // Prepare argument list to generate call. |
| ArgListTy Args; |
| ArgListEntry Entry; |
| Entry.Node = Load; |
| Entry.Ty = CallTy; |
| Args.push_back(Entry); |
| |
| // Setup call to __tls_get_addr. |
| TargetLowering::CallLoweringInfo CLI(DAG); |
| CLI.setDebugLoc(DL) |
| .setChain(DAG.getEntryNode()) |
| .setLibCallee(CallingConv::C, CallTy, |
| DAG.getExternalSymbol("__tls_get_addr", Ty), |
| std::move(Args)); |
| |
| return LowerCallTo(CLI).first; |
| } |
| |
| SDValue |
| LoongArchTargetLowering::lowerGlobalTLSAddress(SDValue Op, |
| SelectionDAG &DAG) const { |
| if (DAG.getMachineFunction().getFunction().getCallingConv() == |
| CallingConv::GHC) |
| report_fatal_error("In GHC calling convention TLS is not supported"); |
| |
| GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op); |
| assert(N->getOffset() == 0 && "unexpected offset in global node"); |
| |
| SDValue Addr; |
| switch (getTargetMachine().getTLSModel(N->getGlobal())) { |
| case TLSModel::GeneralDynamic: |
| // In this model, application code calls the dynamic linker function |
| // __tls_get_addr to locate TLS offsets into the dynamic thread vector at |
| // runtime. |
| Addr = getDynamicTLSAddr(N, DAG, LoongArch::PseudoLA_TLS_GD); |
| break; |
| case TLSModel::LocalDynamic: |
| // Same as GeneralDynamic, except for assembly modifiers and relocation |
| // records. |
| Addr = getDynamicTLSAddr(N, DAG, LoongArch::PseudoLA_TLS_LD); |
| break; |
| case TLSModel::InitialExec: |
| // This model uses the GOT to resolve TLS offsets. |
| Addr = getStaticTLSAddr(N, DAG, LoongArch::PseudoLA_TLS_IE); |
| break; |
| case TLSModel::LocalExec: |
| // This model is used when static linking as the TLS offsets are resolved |
| // during program linking. |
| Addr = getStaticTLSAddr(N, DAG, LoongArch::PseudoLA_TLS_LE); |
| break; |
| } |
| |
| return Addr; |
| } |
| |
| SDValue |
| LoongArchTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op, |
| SelectionDAG &DAG) const { |
| switch (Op.getConstantOperandVal(0)) { |
| default: |
| return SDValue(); // Don't custom lower most intrinsics. |
| case Intrinsic::thread_pointer: { |
| EVT PtrVT = getPointerTy(DAG.getDataLayout()); |
| return DAG.getRegister(LoongArch::R2, PtrVT); |
| } |
| } |
| } |
| |
| // Helper function that emits error message for intrinsics with chain. |
| static SDValue emitIntrinsicWithChainErrorMessage(SDValue Op, |
| StringRef ErrorMsg, |
| SelectionDAG &DAG) { |
| |
| DAG.getContext()->emitError("argument to '" + Op->getOperationName(0) + "' " + |
| ErrorMsg); |
| return DAG.getMergeValues({DAG.getUNDEF(Op.getValueType()), Op.getOperand(0)}, |
| SDLoc(Op)); |
| } |
| |
| SDValue |
| LoongArchTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op, |
| SelectionDAG &DAG) const { |
| SDLoc DL(Op); |
| MVT GRLenVT = Subtarget.getGRLenVT(); |
| SDValue Op0 = Op.getOperand(0); |
| std::string Name = Op->getOperationName(0); |
| const StringRef ErrorMsgOOR = "out of range"; |
| |
| switch (Op.getConstantOperandVal(1)) { |
| default: |
| return Op; |
| case Intrinsic::loongarch_crc_w_b_w: |
| case Intrinsic::loongarch_crc_w_h_w: |
| case Intrinsic::loongarch_crc_w_w_w: |
| case Intrinsic::loongarch_crc_w_d_w: |
| case Intrinsic::loongarch_crcc_w_b_w: |
| case Intrinsic::loongarch_crcc_w_h_w: |
| case Intrinsic::loongarch_crcc_w_w_w: |
| case Intrinsic::loongarch_crcc_w_d_w: { |
| std::string Name = Op->getOperationName(0); |
| DAG.getContext()->emitError(Name + " requires target: loongarch64"); |
| return DAG.getMergeValues({DAG.getUNDEF(Op.getValueType()), Op0}, DL); |
| } |
| case Intrinsic::loongarch_csrrd_w: |
| case Intrinsic::loongarch_csrrd_d: { |
| unsigned Imm = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue(); |
| if (!isUInt<14>(Imm)) |
| return emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG); |
| return DAG.getMergeValues( |
| {DAG.getNode(LoongArchISD::CSRRD, DL, GRLenVT, Op0, |
| DAG.getConstant(Imm, DL, GRLenVT)), |
| Op0}, |
| DL); |
| } |
| case Intrinsic::loongarch_csrwr_w: |
| case Intrinsic::loongarch_csrwr_d: { |
| unsigned Imm = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue(); |
| if (!isUInt<14>(Imm)) |
| return emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG); |
| return DAG.getMergeValues( |
| {DAG.getNode(LoongArchISD::CSRWR, DL, GRLenVT, Op0, Op.getOperand(2), |
| DAG.getConstant(Imm, DL, GRLenVT)), |
| Op0}, |
| DL); |
| } |
| case Intrinsic::loongarch_csrxchg_w: |
| case Intrinsic::loongarch_csrxchg_d: { |
| unsigned Imm = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue(); |
| if (!isUInt<14>(Imm)) |
| return emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG); |
| return DAG.getMergeValues( |
| {DAG.getNode(LoongArchISD::CSRXCHG, DL, GRLenVT, Op0, Op.getOperand(2), |
| Op.getOperand(3), DAG.getConstant(Imm, DL, GRLenVT)), |
| Op0}, |
| DL); |
| } |
| case Intrinsic::loongarch_iocsrrd_d: { |
| if (Subtarget.is64Bit()) |
| return DAG.getMergeValues( |
| {DAG.getNode( |
| LoongArchISD::IOCSRRD_D, DL, GRLenVT, Op0, |
| DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2))), |
| Op0}, |
| DL); |
| else { |
| DAG.getContext()->emitError( |
| "llvm.loongarch.crc.w.d.w requires target: loongarch64"); |
| return DAG.getMergeValues({DAG.getUNDEF(Op.getValueType()), Op0}, DL); |
| } |
| } |
| #define IOCSRRD_CASE(NAME, NODE) \ |
| case Intrinsic::loongarch_##NAME: { \ |
| return DAG.getMergeValues( \ |
| {DAG.getNode(LoongArchISD::NODE, DL, GRLenVT, Op0, Op.getOperand(2)), \ |
| Op0}, \ |
| DL); \ |
| } |
| IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B); |
| IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H); |
| IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W); |
| #undef IOCSRRD_CASE |
| case Intrinsic::loongarch_cpucfg: { |
| return DAG.getMergeValues( |
| {DAG.getNode(LoongArchISD::CPUCFG, DL, GRLenVT, Op0, Op.getOperand(2)), |
| Op0}, |
| DL); |
| } |
| case Intrinsic::loongarch_lddir_d: { |
| unsigned Imm = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue(); |
| if (!isUInt<8>(Imm)) { |
| DAG.getContext()->emitError("argument to '" + Op->getOperationName(0) + |
| "' out of range"); |
| return DAG.getMergeValues({DAG.getUNDEF(Op.getValueType()), Op0}, DL); |
| } |
| |
| return Op; |
| } |
| case Intrinsic::loongarch_movfcsr2gr: { |
| if (!Subtarget.hasBasicF()) { |
| DAG.getContext()->emitError( |
| "llvm.loongarch.movfcsr2gr expects basic f target feature"); |
| return DAG.getMergeValues( |
| {DAG.getUNDEF(Op.getValueType()), Op.getOperand(0)}, SDLoc(Op)); |
| } |
| unsigned Imm = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue(); |
| if (!isUInt<2>(Imm)) { |
| DAG.getContext()->emitError("argument to '" + Op->getOperationName(0) + |
| "' " + ErrorMsgOOR); |
| return DAG.getMergeValues( |
| {DAG.getUNDEF(Op.getValueType()), Op.getOperand(0)}, SDLoc(Op)); |
| } |
| return DAG.getMergeValues( |
| {DAG.getNode(LoongArchISD::MOVFCSR2GR, DL, Op.getValueType(), |
| DAG.getConstant(Imm, DL, GRLenVT)), |
| Op.getOperand(0)}, |
| DL); |
| } |
| } |
| } |
| |
| // Helper function that emits error message for intrinsics with void return |
| // value. |
| static SDValue emitIntrinsicErrorMessage(SDValue Op, StringRef ErrorMsg, |
| SelectionDAG &DAG) { |
| |
| DAG.getContext()->emitError("argument to '" + Op->getOperationName(0) + "' " + |
| ErrorMsg); |
| return Op.getOperand(0); |
| } |
| |
| SDValue LoongArchTargetLowering::lowerINTRINSIC_VOID(SDValue Op, |
| SelectionDAG &DAG) const { |
| SDLoc DL(Op); |
| MVT GRLenVT = Subtarget.getGRLenVT(); |
| SDValue Op0 = Op.getOperand(0); |
| uint64_t IntrinsicEnum = Op.getConstantOperandVal(1); |
| SDValue Op2 = Op.getOperand(2); |
| const StringRef ErrorMsgOOR = "out of range"; |
| |
| switch (IntrinsicEnum) { |
| default: |
| // TODO: Add more Intrinsics. |
| return SDValue(); |
| case Intrinsic::loongarch_cacop_d: |
| case Intrinsic::loongarch_cacop_w: { |
| if (IntrinsicEnum == Intrinsic::loongarch_cacop_d && !Subtarget.is64Bit()) { |
| DAG.getContext()->emitError( |
| "llvm.loongarch.cacop.d requires target: loongarch64"); |
| return Op.getOperand(0); |
| } |
| if (IntrinsicEnum == Intrinsic::loongarch_cacop_w && Subtarget.is64Bit()) { |
| DAG.getContext()->emitError( |
| "llvm.loongarch.cacop.w requires target: loongarch32"); |
| return Op.getOperand(0); |
| } |
| // call void @llvm.loongarch.cacop.[d/w](uimm5, rj, simm12) |
| unsigned Imm1 = cast<ConstantSDNode>(Op2)->getZExtValue(); |
| if (!isUInt<5>(Imm1)) |
| return emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG); |
| SDValue Op4 = Op.getOperand(4); |
| int Imm2 = cast<ConstantSDNode>(Op4)->getSExtValue(); |
| if (!isInt<12>(Imm2)) |
| return emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG); |
| |
| return Op; |
| } |
| |
| case Intrinsic::loongarch_dbar: { |
| unsigned Imm = cast<ConstantSDNode>(Op2)->getZExtValue(); |
| if (!isUInt<15>(Imm)) |
| return emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG); |
| |
| return DAG.getNode(LoongArchISD::DBAR, DL, MVT::Other, Op0, |
| DAG.getConstant(Imm, DL, GRLenVT)); |
| } |
| case Intrinsic::loongarch_ibar: { |
| unsigned Imm = cast<ConstantSDNode>(Op2)->getZExtValue(); |
| if (!isUInt<15>(Imm)) |
| return emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG); |
| |
| return DAG.getNode(LoongArchISD::IBAR, DL, MVT::Other, Op0, |
| DAG.getConstant(Imm, DL, GRLenVT)); |
| } |
| case Intrinsic::loongarch_break: { |
| unsigned Imm = cast<ConstantSDNode>(Op2)->getZExtValue(); |
| if (!isUInt<15>(Imm)) |
| return emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG); |
| |
| return DAG.getNode(LoongArchISD::BREAK, DL, MVT::Other, Op0, |
| DAG.getConstant(Imm, DL, GRLenVT)); |
| } |
| case Intrinsic::loongarch_movgr2fcsr: { |
| if (!Subtarget.hasBasicF()) { |
| DAG.getContext()->emitError( |
| "llvm.loongarch.movgr2fcsr expects basic f target feature"); |
| return Op0; |
| } |
| unsigned Imm = cast<ConstantSDNode>(Op2)->getZExtValue(); |
| if (!isUInt<2>(Imm)) |
| return emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG); |
| |
| return DAG.getNode( |
| LoongArchISD::MOVGR2FCSR, DL, MVT::Other, Op0, |
| DAG.getConstant(Imm, DL, GRLenVT), |
| DAG.getNode(ISD::ANY_EXTEND, DL, GRLenVT, Op.getOperand(3))); |
| } |
| case Intrinsic::loongarch_syscall: { |
| unsigned Imm = cast<ConstantSDNode>(Op2)->getZExtValue(); |
| if (!isUInt<15>(Imm)) |
| return emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG); |
| |
| return DAG.getNode(LoongArchISD::SYSCALL, DL, MVT::Other, Op0, |
| DAG.getConstant(Imm, DL, GRLenVT)); |
| } |
| #define IOCSRWR_CASE(NAME, NODE) \ |
| case Intrinsic::loongarch_##NAME: { \ |
| SDValue Op3 = Op.getOperand(3); \ |
| if (Subtarget.is64Bit()) \ |
| return DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Op0, \ |
| DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \ |
| DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op3)); \ |
| else \ |
| return DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Op0, Op2, Op3); \ |
| } |
| IOCSRWR_CASE(iocsrwr_b, IOCSRWR_B); |
| IOCSRWR_CASE(iocsrwr_h, IOCSRWR_H); |
| IOCSRWR_CASE(iocsrwr_w, IOCSRWR_W); |
| #undef IOCSRWR_CASE |
| case Intrinsic::loongarch_iocsrwr_d: { |
| if (Subtarget.is64Bit()) |
| return DAG.getNode( |
| LoongArchISD::IOCSRWR_D, DL, MVT::Other, Op0, Op2, |
| DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(3))); |
| else { |
| DAG.getContext()->emitError( |
| "llvm.loongarch.iocsrwr.d requires target: loongarch64"); |
| return Op.getOperand(0); |
| } |
| } |
| #define ASRT_LE_GT_CASE(NAME) \ |
| case Intrinsic::loongarch_##NAME: { \ |
| if (!Subtarget.is64Bit()) { \ |
| DAG.getContext()->emitError(Op->getOperationName(0) + \ |
| " requires target: loongarch64"); \ |
| return Op.getOperand(0); \ |
| } \ |
| return Op; \ |
| } |
| ASRT_LE_GT_CASE(asrtle_d) |
| ASRT_LE_GT_CASE(asrtgt_d) |
| #undef ASRT_LE_GT_CASE |
| case Intrinsic::loongarch_ldpte_d: { |
| unsigned Imm = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue(); |
| if (!isUInt<8>(Imm)) |
| return emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG); |
| if (!Subtarget.is64Bit()) { |
| DAG.getContext()->emitError(Op->getOperationName(0) + |
| " requires target: loongarch64"); |
| return Op.getOperand(0); |
| } |
| return Op; |
| } |
| } |
| } |
| |
| SDValue LoongArchTargetLowering::lowerShiftLeftParts(SDValue Op, |
| SelectionDAG &DAG) const { |
| SDLoc DL(Op); |
| SDValue Lo = Op.getOperand(0); |
| SDValue Hi = Op.getOperand(1); |
| SDValue Shamt = Op.getOperand(2); |
| EVT VT = Lo.getValueType(); |
| |
| // if Shamt-GRLen < 0: // Shamt < GRLen |
| // Lo = Lo << Shamt |
| // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (GRLen-1 ^ Shamt)) |
| // else: |
| // Lo = 0 |
| // Hi = Lo << (Shamt-GRLen) |
| |
| SDValue Zero = DAG.getConstant(0, DL, VT); |
| SDValue One = DAG.getConstant(1, DL, VT); |
| SDValue MinusGRLen = DAG.getConstant(-(int)Subtarget.getGRLen(), DL, VT); |
| SDValue GRLenMinus1 = DAG.getConstant(Subtarget.getGRLen() - 1, DL, VT); |
| SDValue ShamtMinusGRLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusGRLen); |
| SDValue GRLenMinus1Shamt = DAG.getNode(ISD::XOR, DL, VT, Shamt, GRLenMinus1); |
| |
| SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt); |
| SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One); |
| SDValue ShiftRightLo = |
| DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, GRLenMinus1Shamt); |
| SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt); |
| SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo); |
| SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusGRLen); |
| |
| SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusGRLen, Zero, ISD::SETLT); |
| |
| Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero); |
| Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse); |
| |
| SDValue Parts[2] = {Lo, Hi}; |
| return DAG.getMergeValues(Parts, DL); |
| } |
| |
| SDValue LoongArchTargetLowering::lowerShiftRightParts(SDValue Op, |
| SelectionDAG &DAG, |
| bool IsSRA) const { |
| SDLoc DL(Op); |
| SDValue Lo = Op.getOperand(0); |
| SDValue Hi = Op.getOperand(1); |
| SDValue Shamt = Op.getOperand(2); |
| EVT VT = Lo.getValueType(); |
| |
| // SRA expansion: |
| // if Shamt-GRLen < 0: // Shamt < GRLen |
| // Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1)) |
| // Hi = Hi >>s Shamt |
| // else: |
| // Lo = Hi >>s (Shamt-GRLen); |
| // Hi = Hi >>s (GRLen-1) |
| // |
| // SRL expansion: |
| // if Shamt-GRLen < 0: // Shamt < GRLen |
| // Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1)) |
| // Hi = Hi >>u Shamt |
| // else: |
| // Lo = Hi >>u (Shamt-GRLen); |
| // Hi = 0; |
| |
| unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL; |
| |
| SDValue Zero = DAG.getConstant(0, DL, VT); |
| SDValue One = DAG.getConstant(1, DL, VT); |
| SDValue MinusGRLen = DAG.getConstant(-(int)Subtarget.getGRLen(), DL, VT); |
| SDValue GRLenMinus1 = DAG.getConstant(Subtarget.getGRLen() - 1, DL, VT); |
| SDValue ShamtMinusGRLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusGRLen); |
| SDValue GRLenMinus1Shamt = DAG.getNode(ISD::XOR, DL, VT, Shamt, GRLenMinus1); |
| |
| SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt); |
| SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One); |
| SDValue ShiftLeftHi = |
| DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, GRLenMinus1Shamt); |
| SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi); |
| SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt); |
| SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusGRLen); |
| SDValue HiFalse = |
| IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, GRLenMinus1) : Zero; |
| |
| SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusGRLen, Zero, ISD::SETLT); |
| |
| Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse); |
| Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse); |
| |
| SDValue Parts[2] = {Lo, Hi}; |
| return DAG.getMergeValues(Parts, DL); |
| } |
| |
| // Returns the opcode of the target-specific SDNode that implements the 32-bit |
| // form of the given Opcode. |
| static LoongArchISD::NodeType getLoongArchWOpcode(unsigned Opcode) { |
| switch (Opcode) { |
| default: |
| llvm_unreachable("Unexpected opcode"); |
| case ISD::SHL: |
| return LoongArchISD::SLL_W; |
| case ISD::SRA: |
| return LoongArchISD::SRA_W; |
| case ISD::SRL: |
| return LoongArchISD::SRL_W; |
| case ISD::ROTR: |
| return LoongArchISD::ROTR_W; |
| case ISD::ROTL: |
| return LoongArchISD::ROTL_W; |
| case ISD::CTTZ: |
| return LoongArchISD::CTZ_W; |
| case ISD::CTLZ: |
| return LoongArchISD::CLZ_W; |
| } |
| } |
| |
| // Converts the given i8/i16/i32 operation to a target-specific SelectionDAG |
| // node. Because i8/i16/i32 isn't a legal type for LA64, these operations would |
| // otherwise be promoted to i64, making it difficult to select the |
| // SLL_W/.../*W later one because the fact the operation was originally of |
| // type i8/i16/i32 is lost. |
| static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, |
| unsigned ExtOpc = ISD::ANY_EXTEND) { |
| SDLoc DL(N); |
| LoongArchISD::NodeType WOpcode = getLoongArchWOpcode(N->getOpcode()); |
| SDValue NewOp0, NewRes; |
| |
| switch (NumOp) { |
| default: |
| llvm_unreachable("Unexpected NumOp"); |
| case 1: { |
| NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0)); |
| NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0); |
| break; |
| } |
| case 2: { |
| NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0)); |
| SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1)); |
| NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1); |
| break; |
| } |
| // TODO:Handle more NumOp. |
| } |
| |
| // ReplaceNodeResults requires we maintain the same type for the return |
| // value. |
| return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes); |
| } |
| |
| void LoongArchTargetLowering::ReplaceNodeResults( |
| SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { |
| SDLoc DL(N); |
| EVT VT = N->getValueType(0); |
| switch (N->getOpcode()) { |
| default: |
| llvm_unreachable("Don't know how to legalize this operation"); |
| case ISD::SHL: |
| case ISD::SRA: |
| case ISD::SRL: |
| case ISD::ROTR: |
| assert(VT == MVT::i32 && Subtarget.is64Bit() && |
| "Unexpected custom legalisation"); |
| if (N->getOperand(1).getOpcode() != ISD::Constant) { |
| Results.push_back(customLegalizeToWOp(N, DAG, 2)); |
| break; |
| } |
| break; |
| case ISD::ROTL: |
| ConstantSDNode *CN; |
| if ((CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))) { |
| Results.push_back(customLegalizeToWOp(N, DAG, 2)); |
| break; |
| } |
| break; |
| case ISD::FP_TO_SINT: { |
| assert(VT == MVT::i32 && Subtarget.is64Bit() && |
| "Unexpected custom legalisation"); |
| SDValue Src = N->getOperand(0); |
| EVT FVT = EVT::getFloatingPointVT(N->getValueSizeInBits(0)); |
| if (getTypeAction(*DAG.getContext(), Src.getValueType()) != |
| TargetLowering::TypeSoftenFloat) { |
| SDValue Dst = DAG.getNode(LoongArchISD::FTINT, DL, FVT, Src); |
| Results.push_back(DAG.getNode(ISD::BITCAST, DL, VT, Dst)); |
| return; |
| } |
| // If the FP type needs to be softened, emit a library call using the 'si' |
| // version. If we left it to default legalization we'd end up with 'di'. |
| RTLIB::Libcall LC; |
| LC = RTLIB::getFPTOSINT(Src.getValueType(), VT); |
| MakeLibCallOptions CallOptions; |
| EVT OpVT = Src.getValueType(); |
| CallOptions.setTypeListBeforeSoften(OpVT, VT, true); |
| SDValue Chain = SDValue(); |
| SDValue Result; |
| std::tie(Result, Chain) = |
| makeLibCall(DAG, LC, VT, Src, CallOptions, DL, Chain); |
| Results.push_back(Result); |
| break; |
| } |
| case ISD::BITCAST: { |
| SDValue Src = N->getOperand(0); |
| EVT SrcVT = Src.getValueType(); |
| if (VT == MVT::i32 && SrcVT == MVT::f32 && Subtarget.is64Bit() && |
| Subtarget.hasBasicF()) { |
| SDValue Dst = |
| DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, MVT::i64, Src); |
| Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Dst)); |
| } |
| break; |
| } |
| case ISD::FP_TO_UINT: { |
| assert(VT == MVT::i32 && Subtarget.is64Bit() && |
| "Unexpected custom legalisation"); |
| auto &TLI = DAG.getTargetLoweringInfo(); |
| SDValue Tmp1, Tmp2; |
| TLI.expandFP_TO_UINT(N, Tmp1, Tmp2, DAG); |
| Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Tmp1)); |
| break; |
| } |
| case ISD::BSWAP: { |
| SDValue Src = N->getOperand(0); |
| assert((VT == MVT::i16 || VT == MVT::i32) && |
| "Unexpected custom legalization"); |
| MVT GRLenVT = Subtarget.getGRLenVT(); |
| SDValue NewSrc = DAG.getNode(ISD::ANY_EXTEND, DL, GRLenVT, Src); |
| SDValue Tmp; |
| switch (VT.getSizeInBits()) { |
| default: |
| llvm_unreachable("Unexpected operand width"); |
| case 16: |
| Tmp = DAG.getNode(LoongArchISD::REVB_2H, DL, GRLenVT, NewSrc); |
| break; |
| case 32: |
| // Only LA64 will get to here due to the size mismatch between VT and |
| // GRLenVT, LA32 lowering is directly defined in LoongArchInstrInfo. |
| Tmp = DAG.getNode(LoongArchISD::REVB_2W, DL, GRLenVT, NewSrc); |
| break; |
| } |
| Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Tmp)); |
| break; |
| } |
| case ISD::BITREVERSE: { |
| SDValue Src = N->getOperand(0); |
| assert((VT == MVT::i8 || (VT == MVT::i32 && Subtarget.is64Bit())) && |
| "Unexpected custom legalization"); |
| MVT GRLenVT = Subtarget.getGRLenVT(); |
| SDValue NewSrc = DAG.getNode(ISD::ANY_EXTEND, DL, GRLenVT, Src); |
| SDValue Tmp; |
| switch (VT.getSizeInBits()) { |
| default: |
| llvm_unreachable("Unexpected operand width"); |
| case 8: |
| Tmp = DAG.getNode(LoongArchISD::BITREV_4B, DL, GRLenVT, NewSrc); |
| break; |
| case 32: |
| Tmp = DAG.getNode(LoongArchISD::BITREV_W, DL, GRLenVT, NewSrc); |
| break; |
| } |
| Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Tmp)); |
| break; |
| } |
| case ISD::CTLZ: |
| case ISD::CTTZ: { |
| assert(VT == MVT::i32 && Subtarget.is64Bit() && |
| "Unexpected custom legalisation"); |
| Results.push_back(customLegalizeToWOp(N, DAG, 1)); |
| break; |
| } |
| case ISD::INTRINSIC_W_CHAIN: { |
| SDValue Op0 = N->getOperand(0); |
| EVT VT = N->getValueType(0); |
| uint64_t Op1 = N->getConstantOperandVal(1); |
| MVT GRLenVT = Subtarget.getGRLenVT(); |
| if (Op1 == Intrinsic::loongarch_movfcsr2gr) { |
| if (!Subtarget.hasBasicF()) { |
| DAG.getContext()->emitError( |
| "llvm.loongarch.movfcsr2gr expects basic f target feature"); |
| Results.push_back(DAG.getMergeValues( |
| {DAG.getUNDEF(N->getValueType(0)), N->getOperand(0)}, SDLoc(N))); |
| Results.push_back(N->getOperand(0)); |
| return; |
| } |
| unsigned Imm = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue(); |
| if (!isUInt<2>(Imm)) { |
| DAG.getContext()->emitError("argument to '" + N->getOperationName(0) + |
| "' " + "out of range"); |
| Results.push_back(DAG.getMergeValues( |
| {DAG.getUNDEF(N->getValueType(0)), N->getOperand(0)}, SDLoc(N))); |
| Results.push_back(N->getOperand(0)); |
| return; |
| } |
| Results.push_back( |
| DAG.getNode(ISD::TRUNCATE, DL, VT, |
| DAG.getNode(LoongArchISD::MOVFCSR2GR, SDLoc(N), MVT::i64, |
| DAG.getConstant(Imm, DL, GRLenVT)))); |
| Results.push_back(N->getOperand(0)); |
| return; |
| } |
| SDValue Op2 = N->getOperand(2); |
| std::string Name = N->getOperationName(0); |
| |
| switch (Op1) { |
| default: |
| llvm_unreachable("Unexpected Intrinsic."); |
| #define CRC_CASE_EXT_BINARYOP(NAME, NODE) \ |
| case Intrinsic::loongarch_##NAME: { \ |
| Results.push_back(DAG.getNode( \ |
| ISD::TRUNCATE, DL, VT, \ |
| DAG.getNode( \ |
| LoongArchISD::NODE, DL, MVT::i64, \ |
| DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \ |
| DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))))); \ |
| Results.push_back(N->getOperand(0)); \ |
| break; \ |
| } |
| CRC_CASE_EXT_BINARYOP(crc_w_b_w, CRC_W_B_W) |
| CRC_CASE_EXT_BINARYOP(crc_w_h_w, CRC_W_H_W) |
| CRC_CASE_EXT_BINARYOP(crc_w_w_w, CRC_W_W_W) |
| CRC_CASE_EXT_BINARYOP(crcc_w_b_w, CRCC_W_B_W) |
| CRC_CASE_EXT_BINARYOP(crcc_w_h_w, CRCC_W_H_W) |
| CRC_CASE_EXT_BINARYOP(crcc_w_w_w, CRCC_W_W_W) |
| #undef CRC_CASE_EXT_BINARYOP |
| |
| #define CRC_CASE_EXT_UNARYOP(NAME, NODE) \ |
| case Intrinsic::loongarch_##NAME: { \ |
| Results.push_back( \ |
| DAG.getNode(ISD::TRUNCATE, DL, VT, \ |
| DAG.getNode(LoongArchISD::NODE, DL, MVT::i64, Op2, \ |
| DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, \ |
| N->getOperand(3))))); \ |
| Results.push_back(N->getOperand(0)); \ |
| break; \ |
| } |
| CRC_CASE_EXT_UNARYOP(crc_w_d_w, CRC_W_D_W) |
| CRC_CASE_EXT_UNARYOP(crcc_w_d_w, CRCC_W_D_W) |
| #undef CRC_CASE_EXT_UNARYOP |
| #define CSR_CASE(ID) \ |
| case Intrinsic::loongarch_##ID: { \ |
| if (!Subtarget.is64Bit()) { \ |
| DAG.getContext()->emitError(Name + " requires target: loongarch64"); \ |
| Results.push_back(DAG.getUNDEF(VT)); \ |
| Results.push_back(N->getOperand(0)); \ |
| } \ |
| break; \ |
| } |
| CSR_CASE(csrrd_d); |
| CSR_CASE(csrwr_d); |
| CSR_CASE(csrxchg_d); |
| CSR_CASE(iocsrrd_d); |
| #undef CSR_CASE |
| case Intrinsic::loongarch_csrrd_w: { |
| unsigned Imm = cast<ConstantSDNode>(Op2)->getZExtValue(); |
| if (!isUInt<14>(Imm)) { |
| DAG.getContext()->emitError("argument to '" + Name + "' out of range"); |
| Results.push_back(DAG.getUNDEF(VT)); |
| Results.push_back(N->getOperand(0)); |
| break; |
| } |
| |
| Results.push_back( |
| DAG.getNode(ISD::TRUNCATE, DL, VT, |
| DAG.getNode(LoongArchISD::CSRRD, DL, GRLenVT, Op0, |
| DAG.getConstant(Imm, DL, GRLenVT)))); |
| Results.push_back(N->getOperand(0)); |
| break; |
| } |
| case Intrinsic::loongarch_csrwr_w: { |
| unsigned Imm = cast<ConstantSDNode>(N->getOperand(3))->getZExtValue(); |
| if (!isUInt<14>(Imm)) { |
| DAG.getContext()->emitError("argument to '" + Name + "' out of range"); |
| Results.push_back(DAG.getUNDEF(VT)); |
| Results.push_back(N->getOperand(0)); |
| break; |
| } |
| |
| Results.push_back(DAG.getNode( |
| ISD::TRUNCATE, DL, VT, |
| DAG.getNode(LoongArchISD::CSRWR, DL, GRLenVT, Op0, |
| DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), |
| DAG.getConstant(Imm, DL, GRLenVT)))); |
| Results.push_back(N->getOperand(0)); |
| break; |
| } |
| case Intrinsic::loongarch_csrxchg_w: { |
| unsigned Imm = cast<ConstantSDNode>(N->getOperand(4))->getZExtValue(); |
| if (!isUInt<14>(Imm)) { |
| DAG.getContext()->emitError("argument to '" + Name + "' out of range"); |
| Results.push_back(DAG.getUNDEF(VT)); |
| Results.push_back(N->getOperand(0)); |
| break; |
| } |
| |
| Results.push_back(DAG.getNode( |
| ISD::TRUNCATE, DL, VT, |
| DAG.getNode( |
| LoongArchISD::CSRXCHG, DL, GRLenVT, Op0, |
| DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), |
| DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3)), |
| DAG.getConstant(Imm, DL, GRLenVT)))); |
| Results.push_back(N->getOperand(0)); |
| break; |
| } |
| #define IOCSRRD_CASE(NAME, NODE) \ |
| case Intrinsic::loongarch_##NAME: { \ |
| Results.push_back(DAG.getNode( \ |
| ISD::TRUNCATE, DL, N->getValueType(0), \ |
| DAG.getNode(LoongArchISD::NODE, DL, MVT::i64, Op0, \ |
| DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2)))); \ |
| Results.push_back(N->getOperand(0)); \ |
| break; \ |
| } |
| IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B); |
| IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H); |
| IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W); |
| #undef IOCSRRD_CASE |
| case Intrinsic::loongarch_cpucfg: { |
| Results.push_back(DAG.getNode( |
| ISD::TRUNCATE, DL, VT, |
| DAG.getNode(LoongArchISD::CPUCFG, DL, GRLenVT, Op0, |
| DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2)))); |
| Results.push_back(Op0); |
| break; |
| } |
| case Intrinsic::loongarch_lddir_d: { |
| if (!Subtarget.is64Bit()) { |
| DAG.getContext()->emitError(N->getOperationName(0) + |
| " requires target: loongarch64"); |
| Results.push_back(DAG.getUNDEF(VT)); |
| Results.push_back(Op0); |
| break; |
| } |
| break; |
| } |
| } |
| break; |
| } |
| case ISD::READ_REGISTER: { |
| if (Subtarget.is64Bit()) |
| DAG.getContext()->emitError( |
| "On LA64, only 64-bit registers can be read."); |
| else |
| DAG.getContext()->emitError( |
| "On LA32, only 32-bit registers can be read."); |
| Results.push_back(DAG.getUNDEF(VT)); |
| Results.push_back(N->getOperand(0)); |
| break; |
| } |
| } |
| } |
| |
| static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG, |
| TargetLowering::DAGCombinerInfo &DCI, |
| const LoongArchSubtarget &Subtarget) { |
| if (DCI.isBeforeLegalizeOps()) |
| return SDValue(); |
| |
| SDValue FirstOperand = N->getOperand(0); |
| SDValue SecondOperand = N->getOperand(1); |
| unsigned FirstOperandOpc = FirstOperand.getOpcode(); |
| EVT ValTy = N->getValueType(0); |
| SDLoc DL(N); |
| uint64_t lsb, msb; |
| unsigned SMIdx, SMLen; |
| ConstantSDNode *CN; |
| SDValue NewOperand; |
| MVT GRLenVT = Subtarget.getGRLenVT(); |
| |
| // Op's second operand must be a shifted mask. |
| if (!(CN = dyn_cast<ConstantSDNode>(SecondOperand)) || |
| !isShiftedMask_64(CN->getZExtValue(), SMIdx, SMLen)) |
| return SDValue(); |
| |
| if (FirstOperandOpc == ISD::SRA || FirstOperandOpc == ISD::SRL) { |
| // Pattern match BSTRPICK. |
| // $dst = and ((sra or srl) $src , lsb), (2**len - 1) |
| // => BSTRPICK $dst, $src, msb, lsb |
| // where msb = lsb + len - 1 |
| |
| // The second operand of the shift must be an immediate. |
| if (!(CN = dyn_cast<ConstantSDNode>(FirstOperand.getOperand(1)))) |
| return SDValue(); |
| |
| lsb = CN->getZExtValue(); |
| |
| // Return if the shifted mask does not start at bit 0 or the sum of its |
| // length and lsb exceeds the word's size. |
| if (SMIdx != 0 || lsb + SMLen > ValTy.getSizeInBits()) |
| return SDValue(); |
| |
| NewOperand = FirstOperand.getOperand(0); |
| } else { |
| // Pattern match BSTRPICK. |
| // $dst = and $src, (2**len- 1) , if len > 12 |
| // => BSTRPICK $dst, $src, msb, lsb |
| // where lsb = 0 and msb = len - 1 |
| |
| // If the mask is <= 0xfff, andi can be used instead. |
| if (CN->getZExtValue() <= 0xfff) |
| return SDValue(); |
| |
| // Return if the mask doesn't start at position 0. |
| if (SMIdx) |
| return SDValue(); |
| |
| lsb = 0; |
| NewOperand = FirstOperand; |
| } |
| msb = lsb + SMLen - 1; |
| return DAG.getNode(LoongArchISD::BSTRPICK, DL, ValTy, NewOperand, |
| DAG.getConstant(msb, DL, GRLenVT), |
| DAG.getConstant(lsb, DL, GRLenVT)); |
| } |
| |
| static SDValue performSRLCombine(SDNode *N, SelectionDAG &DAG, |
| TargetLowering::DAGCombinerInfo &DCI, |
| const LoongArchSubtarget &Subtarget) { |
| if (DCI.isBeforeLegalizeOps()) |
| return SDValue(); |
| |
| // $dst = srl (and $src, Mask), Shamt |
| // => |
| // BSTRPICK $dst, $src, MaskIdx+MaskLen-1, Shamt |
| // when Mask is a shifted mask, and MaskIdx <= Shamt <= MaskIdx+MaskLen-1 |
| // |
| |
| SDValue FirstOperand = N->getOperand(0); |
| ConstantSDNode *CN; |
| EVT ValTy = N->getValueType(0); |
| SDLoc DL(N); |
| MVT GRLenVT = Subtarget.getGRLenVT(); |
| unsigned MaskIdx, MaskLen; |
| uint64_t Shamt; |
| |
| // The first operand must be an AND and the second operand of the AND must be |
| // a shifted mask. |
| if (FirstOperand.getOpcode() != ISD::AND || |
| !(CN = dyn_cast<ConstantSDNode>(FirstOperand.getOperand(1))) || |
| !isShiftedMask_64(CN->getZExtValue(), MaskIdx, MaskLen)) |
| return SDValue(); |
| |
| // The second operand (shift amount) must be an immediate. |
| if (!(CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))) |
| return SDValue(); |
| |
| Shamt = CN->getZExtValue(); |
| if (MaskIdx <= Shamt && Shamt <= MaskIdx + MaskLen - 1) |
| return DAG.getNode(LoongArchISD::BSTRPICK, DL, ValTy, |
| FirstOperand->getOperand(0), |
| DAG.getConstant(MaskIdx + MaskLen - 1, DL, GRLenVT), |
| DAG.getConstant(Shamt, DL, GRLenVT)); |
| |
| return SDValue(); |
| } |
| |
| static SDValue performORCombine(SDNode *N, SelectionDAG &DAG, |
| TargetLowering::DAGCombinerInfo &DCI, |
| const LoongArchSubtarget &Subtarget) { |
| MVT GRLenVT = Subtarget.getGRLenVT(); |
| EVT ValTy = N->getValueType(0); |
| SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); |
| ConstantSDNode *CN0, *CN1; |
| SDLoc DL(N); |
| unsigned ValBits = ValTy.getSizeInBits(); |
| unsigned MaskIdx0, MaskLen0, MaskIdx1, MaskLen1; |
| unsigned Shamt; |
| bool SwapAndRetried = false; |
| |
| if (DCI.isBeforeLegalizeOps()) |
| return SDValue(); |
| |
| if (ValBits != 32 && ValBits != 64) |
| return SDValue(); |
| |
| Retry: |
| // 1st pattern to match BSTRINS: |
| // R = or (and X, mask0), (and (shl Y, lsb), mask1) |
| // where mask1 = (2**size - 1) << lsb, mask0 = ~mask1 |
| // => |
| // R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1) |
| if (N0.getOpcode() == ISD::AND && |
| (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) && |
| isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) && |
| N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL && |
| (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(1))) && |
| isShiftedMask_64(CN1->getZExtValue(), MaskIdx1, MaskLen1) && |
| MaskIdx0 == MaskIdx1 && MaskLen0 == MaskLen1 && |
| (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(1))) && |
| (Shamt = CN1->getZExtValue()) == MaskIdx0 && |
| (MaskIdx0 + MaskLen0 <= ValBits)) { |
| LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 1\n"); |
| return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0), |
| N1.getOperand(0).getOperand(0), |
| DAG.getConstant((MaskIdx0 + MaskLen0 - 1), DL, GRLenVT), |
| DAG.getConstant(MaskIdx0, DL, GRLenVT)); |
| } |
| |
| // 2nd pattern to match BSTRINS: |
| // R = or (and X, mask0), (shl (and Y, mask1), lsb) |
| // where mask1 = (2**size - 1), mask0 = ~(mask1 << lsb) |
| // => |
| // R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1) |
| if (N0.getOpcode() == ISD::AND && |
| (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) && |
| isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) && |
| N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::AND && |
| (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(1))) && |
| (Shamt = CN1->getZExtValue()) == MaskIdx0 && |
| (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(1))) && |
| isShiftedMask_64(CN1->getZExtValue(), MaskIdx1, MaskLen1) && |
| MaskLen0 == MaskLen1 && MaskIdx1 == 0 && |
| (MaskIdx0 + MaskLen0 <= ValBits)) { |
| LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 2\n"); |
| return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0), |
| N1.getOperand(0).getOperand(0), |
| DAG.getConstant((MaskIdx0 + MaskLen0 - 1), DL, GRLenVT), |
| DAG.getConstant(MaskIdx0, DL, GRLenVT)); |
| } |
| |
| // 3rd pattern to match BSTRINS: |
| // R = or (and X, mask0), (and Y, mask1) |
| // where ~mask0 = (2**size - 1) << lsb, mask0 & mask1 = 0 |
| // => |
| // R = BSTRINS X, (shr (and Y, mask1), lsb), msb, lsb |
| // where msb = lsb + size - 1 |
| if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND && |
| (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) && |
| isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) && |
| (MaskIdx0 + MaskLen0 <= 64) && |
| (CN1 = dyn_cast<ConstantSDNode>(N1->getOperand(1))) && |
| (CN1->getSExtValue() & CN0->getSExtValue()) == 0) { |
| LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 3\n"); |
| return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0), |
| DAG.getNode(ISD::SRL, DL, N1->getValueType(0), N1, |
| DAG.getConstant(MaskIdx0, DL, GRLenVT)), |
| DAG.getConstant(ValBits == 32 |
| ? (MaskIdx0 + (MaskLen0 & 31) - 1) |
| : (MaskIdx0 + MaskLen0 - 1), |
| DL, GRLenVT), |
| DAG.getConstant(MaskIdx0, DL, GRLenVT)); |
| } |
| |
| // 4th pattern to match BSTRINS: |
| // R = or (and X, mask), (shl Y, shamt) |
| // where mask = (2**shamt - 1) |
| // => |
| // R = BSTRINS X, Y, ValBits - 1, shamt |
| // where ValBits = 32 or 64 |
| if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::SHL && |
| (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) && |
| isShiftedMask_64(CN0->getZExtValue(), MaskIdx0, MaskLen0) && |
| MaskIdx0 == 0 && (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(1))) && |
| (Shamt = CN1->getZExtValue()) == MaskLen0 && |
| (MaskIdx0 + MaskLen0 <= ValBits)) { |
| LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 4\n"); |
| return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0), |
| N1.getOperand(0), |
| DAG.getConstant((ValBits - 1), DL, GRLenVT), |
| DAG.getConstant(Shamt, DL, GRLenVT)); |
| } |
| |
| // 5th pattern to match BSTRINS: |
| // R = or (and X, mask), const |
| // where ~mask = (2**size - 1) << lsb, mask & const = 0 |
| // => |
| // R = BSTRINS X, (const >> lsb), msb, lsb |
| // where msb = lsb + size - 1 |
| if (N0.getOpcode() == ISD::AND && |
| (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) && |
| isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) && |
| (CN1 = dyn_cast<ConstantSDNode>(N1)) && |
| (CN1->getSExtValue() & CN0->getSExtValue()) == 0) { |
| LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 5\n"); |
| return DAG.getNode( |
| LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0), |
| DAG.getConstant(CN1->getSExtValue() >> MaskIdx0, DL, ValTy), |
| DAG.getConstant((MaskIdx0 + MaskLen0 - 1), DL, GRLenVT), |
| DAG.getConstant(MaskIdx0, DL, GRLenVT)); |
| } |
| |
| // 6th pattern. |
| // a = b | ((c & mask) << shamt), where all positions in b to be overwritten |
| // by the incoming bits are known to be zero. |
| // => |
| // a = BSTRINS b, c, shamt + MaskLen - 1, shamt |
| // |
| // Note that the 1st pattern is a special situation of the 6th, i.e. the 6th |
| // pattern is more common than the 1st. So we put the 1st before the 6th in |
| // order to match as many nodes as possible. |
| ConstantSDNode *CNMask, *CNShamt; |
| unsigned MaskIdx, MaskLen; |
| if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::AND && |
| (CNMask = dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(1))) && |
| isShiftedMask_64(CNMask->getZExtValue(), MaskIdx, MaskLen) && |
| MaskIdx == 0 && (CNShamt = dyn_cast<ConstantSDNode>(N1.getOperand(1))) && |
| CNShamt->getZExtValue() + MaskLen <= ValBits) { |
| Shamt = CNShamt->getZExtValue(); |
| APInt ShMask(ValBits, CNMask->getZExtValue() << Shamt); |
| if (ShMask.isSubsetOf(DAG.computeKnownBits(N0).Zero)) { |
| LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 6\n"); |
| return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0, |
| N1.getOperand(0).getOperand(0), |
| DAG.getConstant(Shamt + MaskLen - 1, DL, GRLenVT), |
| DAG.getConstant(Shamt, DL, GRLenVT)); |
| } |
| } |
| |
| // 7th pattern. |
| // a = b | ((c << shamt) & shifted_mask), where all positions in b to be |
| // overwritten by the incoming bits are known to be zero. |
| // => |
| // a = BSTRINS b, c, MaskIdx + MaskLen - 1, MaskIdx |
| // |
| // Similarly, the 7th pattern is more common than the 2nd. So we put the 2nd |
| // before the 7th in order to match as many nodes as possible. |
| if (N1.getOpcode() == ISD::AND && |
| (CNMask = dyn_cast<ConstantSDNode>(N1.getOperand(1))) && |
| isShiftedMask_64(CNMask->getZExtValue(), MaskIdx, MaskLen) && |
| N1.getOperand(0).getOpcode() == ISD::SHL && |
| (CNShamt = dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(1))) && |
| CNShamt->getZExtValue() == MaskIdx) { |
| APInt ShMask(ValBits, CNMask->getZExtValue()); |
| if (ShMask.isSubsetOf(DAG.computeKnownBits(N0).Zero)) { |
| LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 7\n"); |
| return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0, |
| N1.getOperand(0).getOperand(0), |
| DAG.getConstant(MaskIdx + MaskLen - 1, DL, GRLenVT), |
| DAG.getConstant(MaskIdx, DL, GRLenVT)); |
| } |
| } |
| |
| // (or a, b) and (or b, a) are equivalent, so swap the operands and retry. |
| if (!SwapAndRetried) { |
| std::swap(N0, N1); |
| SwapAndRetried = true; |
| goto Retry; |
| } |
| |
| SwapAndRetried = false; |
| Retry2: |
| // 8th pattern. |
| // a = b | (c & shifted_mask), where all positions in b to be overwritten by |
| // the incoming bits are known to be zero. |
| // => |
| // a = BSTRINS b, c >> MaskIdx, MaskIdx + MaskLen - 1, MaskIdx |
| // |
| // Similarly, the 8th pattern is more common than the 4th and 5th patterns. So |
| // we put it here in order to match as many nodes as possible or generate less |
| // instructions. |
| if (N1.getOpcode() == ISD::AND && |
| (CNMask = dyn_cast<ConstantSDNode>(N1.getOperand(1))) && |
| isShiftedMask_64(CNMask->getZExtValue(), MaskIdx, MaskLen)) { |
| APInt ShMask(ValBits, CNMask->getZExtValue()); |
| if (ShMask.isSubsetOf(DAG.computeKnownBits(N0).Zero)) { |
| LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 8\n"); |
| return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0, |
| DAG.getNode(ISD::SRL, DL, N1->getValueType(0), |
| N1->getOperand(0), |
| DAG.getConstant(MaskIdx, DL, GRLenVT)), |
| DAG.getConstant(MaskIdx + MaskLen - 1, DL, GRLenVT), |
| DAG.getConstant(MaskIdx, DL, GRLenVT)); |
| } |
| } |
| // Swap N0/N1 and retry. |
| if (!SwapAndRetried) { |
| std::swap(N0, N1); |
| SwapAndRetried = true; |
| goto Retry2; |
| } |
| |
| return SDValue(); |
| } |
| |
| // Combine (loongarch_bitrev_w (loongarch_revb_2w X)) to loongarch_bitrev_4b. |
| static SDValue performBITREV_WCombine(SDNode *N, SelectionDAG &DAG, |
| TargetLowering::DAGCombinerInfo &DCI, |
| const LoongArchSubtarget &Subtarget) { |
| if (DCI.isBeforeLegalizeOps()) |
| return SDValue(); |
| |
| SDValue Src = N->getOperand(0); |
| if (Src.getOpcode() != LoongArchISD::REVB_2W) |
| return SDValue(); |
| |
| return DAG.getNode(LoongArchISD::BITREV_4B, SDLoc(N), N->getValueType(0), |
| Src.getOperand(0)); |
| } |
| |
| SDValue LoongArchTargetLowering::PerformDAGCombine(SDNode *N, |
| DAGCombinerInfo &DCI) const { |
| SelectionDAG &DAG = DCI.DAG; |
| switch (N->getOpcode()) { |
| default: |
| break; |
| case ISD::AND: |
| return performANDCombine(N, DAG, DCI, Subtarget); |
| case ISD::OR: |
| return performORCombine(N, DAG, DCI, Subtarget); |
| case ISD::SRL: |
| return performSRLCombine(N, DAG, DCI, Subtarget); |
| case LoongArchISD::BITREV_W: |
| return performBITREV_WCombine(N, DAG, DCI, Subtarget); |
| } |
| return SDValue(); |
| } |
| |
| static MachineBasicBlock *insertDivByZeroTrap(MachineInstr &MI, |
| MachineBasicBlock *MBB) { |
| if (!ZeroDivCheck) |
| return MBB; |
| |
| // Build instructions: |
| // MBB: |
| // div(or mod) $dst, $dividend, $divisor |
| // bnez $divisor, SinkMBB |
| // BreakMBB: |
| // break 7 // BRK_DIVZERO |
| // SinkMBB: |
| // fallthrough |
| const BasicBlock *LLVM_BB = MBB->getBasicBlock(); |
| MachineFunction::iterator It = ++MBB->getIterator(); |
| MachineFunction *MF = MBB->getParent(); |
| auto BreakMBB = MF->CreateMachineBasicBlock(LLVM_BB); |
| auto SinkMBB = MF->CreateMachineBasicBlock(LLVM_BB); |
| MF->insert(It, BreakMBB); |
| MF->insert(It, SinkMBB); |
| |
| // Transfer the remainder of MBB and its successor edges to SinkMBB. |
| SinkMBB->splice(SinkMBB->end(), MBB, std::next(MI.getIterator()), MBB->end()); |
| SinkMBB->transferSuccessorsAndUpdatePHIs(MBB); |
| |
| const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo(); |
| DebugLoc DL = MI.getDebugLoc(); |
| MachineOperand &Divisor = MI.getOperand(2); |
| Register DivisorReg = Divisor.getReg(); |
| |
| // MBB: |
| BuildMI(MBB, DL, TII.get(LoongArch::BNEZ)) |
| .addReg(DivisorReg, getKillRegState(Divisor.isKill())) |
| .addMBB(SinkMBB); |
| MBB->addSuccessor(BreakMBB); |
| MBB->addSuccessor(SinkMBB); |
| |
| // BreakMBB: |
| // See linux header file arch/loongarch/include/uapi/asm/break.h for the |
| // definition of BRK_DIVZERO. |
| BuildMI(BreakMBB, DL, TII.get(LoongArch::BREAK)).addImm(7 /*BRK_DIVZERO*/); |
| BreakMBB->addSuccessor(SinkMBB); |
| |
| // Clear Divisor's kill flag. |
| Divisor.setIsKill(false); |
| |
| return SinkMBB; |
| } |
| |
| MachineBasicBlock *LoongArchTargetLowering::EmitInstrWithCustomInserter( |
| MachineInstr &MI, MachineBasicBlock *BB) const { |
| const TargetInstrInfo *TII = Subtarget.getInstrInfo(); |
| DebugLoc DL = MI.getDebugLoc(); |
| |
| switch (MI.getOpcode()) { |
| default: |
| llvm_unreachable("Unexpected instr type to insert"); |
| case LoongArch::DIV_W: |
| case LoongArch::DIV_WU: |
| case LoongArch::MOD_W: |
| case LoongArch::MOD_WU: |
| case LoongArch::DIV_D: |
| case LoongArch::DIV_DU: |
| case LoongArch::MOD_D: |
| case LoongArch::MOD_DU: |
| return insertDivByZeroTrap(MI, BB); |
| break; |
| case LoongArch::WRFCSR: { |
| BuildMI(*BB, MI, DL, TII->get(LoongArch::MOVGR2FCSR), |
| LoongArch::FCSR0 + MI.getOperand(0).getImm()) |
| .addReg(MI.getOperand(1).getReg()); |
| MI.eraseFromParent(); |
| return BB; |
| } |
| case LoongArch::RDFCSR: { |
| MachineInstr *ReadFCSR = |
| BuildMI(*BB, MI, DL, TII->get(LoongArch::MOVFCSR2GR), |
| MI.getOperand(0).getReg()) |
| .addReg(LoongArch::FCSR0 + MI.getOperand(1).getImm()); |
| ReadFCSR->getOperand(1).setIsUndef(); |
| MI.eraseFromParent(); |
| return BB; |
| } |
| } |
| } |
| |
| const char *LoongArchTargetLowering::getTargetNodeName(unsigned Opcode) const { |
| switch ((LoongArchISD::NodeType)Opcode) { |
| case LoongArchISD::FIRST_NUMBER: |
| break; |
| |
| #define NODE_NAME_CASE(node) \ |
| case LoongArchISD::node: \ |
| return "LoongArchISD::" #node; |
| |
| // TODO: Add more target-dependent nodes later. |
| NODE_NAME_CASE(CALL) |
| NODE_NAME_CASE(RET) |
| NODE_NAME_CASE(TAIL) |
| NODE_NAME_CASE(SLL_W) |
| NODE_NAME_CASE(SRA_W) |
| NODE_NAME_CASE(SRL_W) |
| NODE_NAME_CASE(BSTRINS) |
| NODE_NAME_CASE(BSTRPICK) |
| NODE_NAME_CASE(MOVGR2FR_W_LA64) |
| NODE_NAME_CASE(MOVFR2GR_S_LA64) |
| NODE_NAME_CASE(FTINT) |
| NODE_NAME_CASE(REVB_2H) |
| NODE_NAME_CASE(REVB_2W) |
| NODE_NAME_CASE(BITREV_4B) |
| NODE_NAME_CASE(BITREV_W) |
| NODE_NAME_CASE(ROTR_W) |
| NODE_NAME_CASE(ROTL_W) |
| NODE_NAME_CASE(CLZ_W) |
| NODE_NAME_CASE(CTZ_W) |
| NODE_NAME_CASE(DBAR) |
| NODE_NAME_CASE(IBAR) |
| NODE_NAME_CASE(BREAK) |
| NODE_NAME_CASE(SYSCALL) |
| NODE_NAME_CASE(CRC_W_B_W) |
| NODE_NAME_CASE(CRC_W_H_W) |
| NODE_NAME_CASE(CRC_W_W_W) |
| NODE_NAME_CASE(CRC_W_D_W) |
| NODE_NAME_CASE(CRCC_W_B_W) |
| NODE_NAME_CASE(CRCC_W_H_W) |
| NODE_NAME_CASE(CRCC_W_W_W) |
| NODE_NAME_CASE(CRCC_W_D_W) |
| NODE_NAME_CASE(CSRRD) |
| NODE_NAME_CASE(CSRWR) |
| NODE_NAME_CASE(CSRXCHG) |
| NODE_NAME_CASE(IOCSRRD_B) |
| NODE_NAME_CASE(IOCSRRD_H) |
| NODE_NAME_CASE(IOCSRRD_W) |
| NODE_NAME_CASE(IOCSRRD_D) |
| NODE_NAME_CASE(IOCSRWR_B) |
| NODE_NAME_CASE(IOCSRWR_H) |
| NODE_NAME_CASE(IOCSRWR_W) |
| NODE_NAME_CASE(IOCSRWR_D) |
| NODE_NAME_CASE(CPUCFG) |
| NODE_NAME_CASE(MOVGR2FCSR) |
| NODE_NAME_CASE(MOVFCSR2GR) |
| NODE_NAME_CASE(CACOP_D) |
| NODE_NAME_CASE(CACOP_W) |
| } |
| #undef NODE_NAME_CASE |
| return nullptr; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Calling Convention Implementation |
| //===----------------------------------------------------------------------===// |
| |
| // Eight general-purpose registers a0-a7 used for passing integer arguments, |
| // with a0-a1 reused to return values. Generally, the GPRs are used to pass |
| // fixed-point arguments, and floating-point arguments when no FPR is available |
| // or with soft float ABI. |
| const MCPhysReg ArgGPRs[] = {LoongArch::R4, LoongArch::R5, LoongArch::R6, |
| LoongArch::R7, LoongArch::R8, LoongArch::R9, |
| LoongArch::R10, LoongArch::R11}; |
| // Eight floating-point registers fa0-fa7 used for passing floating-point |
| // arguments, and fa0-fa1 are also used to return values. |
| const MCPhysReg ArgFPR32s[] = {LoongArch::F0, LoongArch::F1, LoongArch::F2, |
| LoongArch::F3, LoongArch::F4, LoongArch::F5, |
| LoongArch::F6, LoongArch::F7}; |
| // FPR32 and FPR64 alias each other. |
| const MCPhysReg ArgFPR64s[] = { |
| LoongArch::F0_64, LoongArch::F1_64, LoongArch::F2_64, LoongArch::F3_64, |
| LoongArch::F4_64, LoongArch::F5_64, LoongArch::F6_64, LoongArch::F7_64}; |
| |
| // Pass a 2*GRLen argument that has been split into two GRLen values through |
| // registers or the stack as necessary. |
| static bool CC_LoongArchAssign2GRLen(unsigned GRLen, CCState &State, |
| CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, |
| unsigned ValNo2, MVT ValVT2, MVT LocVT2, |
| ISD::ArgFlagsTy ArgFlags2) { |
| unsigned GRLenInBytes = GRLen / 8; |
| if (Register Reg = State.AllocateReg(ArgGPRs)) { |
| // At least one half can be passed via register. |
| State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg, |
| VA1.getLocVT(), CCValAssign::Full)); |
| } else { |
| // Both halves must be passed on the stack, with proper alignment. |
| Align StackAlign = |
| std::max(Align(GRLenInBytes), ArgFlags1.getNonZeroOrigAlign()); |
| State.addLoc( |
| CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(), |
| State.AllocateStack(GRLenInBytes, StackAlign), |
| VA1.getLocVT(), CCValAssign::Full)); |
| State.addLoc(CCValAssign::getMem( |
| ValNo2, ValVT2, State.AllocateStack(GRLenInBytes, Align(GRLenInBytes)), |
| LocVT2, CCValAssign::Full)); |
| return false; |
| } |
| if (Register Reg = State.AllocateReg(ArgGPRs)) { |
| // The second half can also be passed via register. |
| State.addLoc( |
| CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full)); |
| } else { |
| // The second half is passed via the stack, without additional alignment. |
| State.addLoc(CCValAssign::getMem( |
| ValNo2, ValVT2, State.AllocateStack(GRLenInBytes, Align(GRLenInBytes)), |
| LocVT2, CCValAssign::Full)); |
| } |
| return false; |
| } |
| |
| // Implements the LoongArch calling convention. Returns true upon failure. |
| static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI, |
| unsigned ValNo, MVT ValVT, |
| CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, |
| CCState &State, bool IsFixed, bool IsRet, |
| Type *OrigTy) { |
| unsigned GRLen = DL.getLargestLegalIntTypeSizeInBits(); |
| assert((GRLen == 32 || GRLen == 64) && "Unspport GRLen"); |
| MVT GRLenVT = GRLen == 32 ? MVT::i32 : MVT::i64; |
| MVT LocVT = ValVT; |
| |
| // Any return value split into more than two values can't be returned |
| // directly. |
| if (IsRet && ValNo > 1) |
| return true; |
| |
| // If passing a variadic argument, or if no FPR is available. |
| bool UseGPRForFloat = true; |
| |
| switch (ABI) { |
| default: |
| llvm_unreachable("Unexpected ABI"); |
| case LoongArchABI::ABI_ILP32S: |
| case LoongArchABI::ABI_LP64S: |
| case LoongArchABI::ABI_ILP32F: |
| case LoongArchABI::ABI_LP64F: |
| report_fatal_error("Unimplemented ABI"); |
| break; |
| case LoongArchABI::ABI_ILP32D: |
| case LoongArchABI::ABI_LP64D: |
| UseGPRForFloat = !IsFixed; |
| break; |
| } |
| |
| // FPR32 and FPR64 alias each other. |
| if (State.getFirstUnallocated(ArgFPR32s) == std::size(ArgFPR32s)) |
| UseGPRForFloat = true; |
| |
| if (UseGPRForFloat && ValVT == MVT::f32) { |
| LocVT = GRLenVT; |
| LocInfo = CCValAssign::BCvt; |
| } else if (UseGPRForFloat && GRLen == 64 && ValVT == MVT::f64) { |
| LocVT = MVT::i64; |
| LocInfo = CCValAssign::BCvt; |
| } else if (UseGPRForFloat && GRLen == 32 && ValVT == MVT::f64) { |
| // TODO: Handle passing f64 on LA32 with D feature. |
| report_fatal_error("Passing f64 with GPR on LA32 is undefined"); |
| } |
| |
| // If this is a variadic argument, the LoongArch calling convention requires |
| // that it is assigned an 'even' or 'aligned' register if it has (2*GRLen)/8 |
| // byte alignment. An aligned register should be used regardless of whether |
| // the original argument was split during legalisation or not. The argument |
| // will not be passed by registers if the original type is larger than |
| // 2*GRLen, so the register alignment rule does not apply. |
| unsigned TwoGRLenInBytes = (2 * GRLen) / 8; |
| if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoGRLenInBytes && |
| DL.getTypeAllocSize(OrigTy) == TwoGRLenInBytes) { |
| unsigned RegIdx = State.getFirstUnallocated(ArgGPRs); |
| // Skip 'odd' register if necessary. |
| if (RegIdx != std::size(ArgGPRs) && RegIdx % 2 == 1) |
| State.AllocateReg(ArgGPRs); |
| } |
| |
| SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs(); |
| SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags = |
| State.getPendingArgFlags(); |
| |
| assert(PendingLocs.size() == PendingArgFlags.size() && |
| "PendingLocs and PendingArgFlags out of sync"); |
| |
| // Split arguments might be passed indirectly, so keep track of the pending |
| // values. |
| if (ValVT.isScalarInteger() && (ArgFlags.isSplit() || !PendingLocs.empty())) { |
| LocVT = GRLenVT; |
| LocInfo = CCValAssign::Indirect; |
| PendingLocs.push_back( |
| CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo)); |
| PendingArgFlags.push_back(ArgFlags); |
| if (!ArgFlags.isSplitEnd()) { |
| return false; |
| } |
| } |
| |
| // If the split argument only had two elements, it should be passed directly |
| // in registers or on the stack. |
| if (ValVT.isScalarInteger() && ArgFlags.isSplitEnd() && |
| PendingLocs.size() <= 2) { |
| assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()"); |
| // Apply the normal calling convention rules to the first half of the |
| // split argument. |
| CCValAssign VA = PendingLocs[0]; |
| ISD::ArgFlagsTy AF = PendingArgFlags[0]; |
| PendingLocs.clear(); |
| PendingArgFlags.clear(); |
| return CC_LoongArchAssign2GRLen(GRLen, State, VA, AF, ValNo, ValVT, LocVT, |
| ArgFlags); |
| } |
| |
| // Allocate to a register if possible, or else a stack slot. |
| Register Reg; |
| unsigned StoreSizeBytes = GRLen / 8; |
| Align StackAlign = Align(GRLen / 8); |
| |
| if (ValVT == MVT::f32 && !UseGPRForFloat) |
| Reg = State.AllocateReg(ArgFPR32s); |
| else if (ValVT == MVT::f64 && !UseGPRForFloat) |
| Reg = State.AllocateReg(ArgFPR64s); |
| else |
| Reg = State.AllocateReg(ArgGPRs); |
| |
| unsigned StackOffset = |
| Reg ? 0 : State.AllocateStack(StoreSizeBytes, StackAlign); |
| |
| // If we reach this point and PendingLocs is non-empty, we must be at the |
| // end of a split argument that must be passed indirectly. |
| if (!PendingLocs.empty()) { |
| assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()"); |
| assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()"); |
| for (auto &It : PendingLocs) { |
| if (Reg) |
| It.convertToReg(Reg); |
| else |
| It.convertToMem(StackOffset); |
| State.addLoc(It); |
| } |
| PendingLocs.clear(); |
| PendingArgFlags.clear(); |
| return false; |
| } |
| assert((!UseGPRForFloat || LocVT == GRLenVT) && |
| "Expected an GRLenVT at this stage"); |
| |
| if (Reg) { |
| State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); |
| return false; |
| } |
| |
| // When a floating-point value is passed on the stack, no bit-cast is needed. |
| if (ValVT.isFloatingPoint()) { |
| LocVT = ValVT; |
| LocInfo = CCValAssign::Full; |
| } |
| |
| State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo)); |
| return false; |
| } |
| |
| void LoongArchTargetLowering::analyzeInputArgs( |
| MachineFunction &MF, CCState &CCInfo, |
| const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet, |
| LoongArchCCAssignFn Fn) const { |
| FunctionType *FType = MF.getFunction().getFunctionType(); |
| for (unsigned i = 0, e = Ins.size(); i != e; ++i) { |
| MVT ArgVT = Ins[i].VT; |
| Type *ArgTy = nullptr; |
| if (IsRet) |
| ArgTy = FType->getReturnType(); |
| else if (Ins[i].isOrigArg()) |
| ArgTy = FType->getParamType(Ins[i].getOrigArgIndex()); |
| LoongArchABI::ABI ABI = |
| MF.getSubtarget<LoongArchSubtarget>().getTargetABI(); |
| if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Ins[i].Flags, |
| CCInfo, /*IsFixed=*/true, IsRet, ArgTy)) { |
| LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type " |
| << EVT(ArgVT).getEVTString() << '\n'); |
| llvm_unreachable(""); |
| } |
| } |
| } |
| |
| void LoongArchTargetLowering::analyzeOutputArgs( |
| MachineFunction &MF, CCState &CCInfo, |
| const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet, |
| CallLoweringInfo *CLI, LoongArchCCAssignFn Fn) const { |
| for (unsigned i = 0, e = Outs.size(); i != e; ++i) { |
| MVT ArgVT = Outs[i].VT; |
| Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr; |
| LoongArchABI::ABI ABI = |
| MF.getSubtarget<LoongArchSubtarget>().getTargetABI(); |
| if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Outs[i].Flags, |
| CCInfo, Outs[i].IsFixed, IsRet, OrigTy)) { |
| LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type " |
| << EVT(ArgVT).getEVTString() << "\n"); |
| llvm_unreachable(""); |
| } |
| } |
| } |
| |
| // Convert Val to a ValVT. Should not be called for CCValAssign::Indirect |
| // values. |
| static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, |
| const CCValAssign &VA, const SDLoc &DL) { |
| switch (VA.getLocInfo()) { |
| default: |
| llvm_unreachable("Unexpected CCValAssign::LocInfo"); |
| case CCValAssign::Full: |
| case CCValAssign::Indirect: |
| break; |
| case CCValAssign::BCvt: |
| if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) |
| Val = DAG.getNode(LoongArchISD::MOVGR2FR_W_LA64, DL, MVT::f32, Val); |
| else |
| Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val); |
| break; |
| } |
| return Val; |
| } |
| |
| static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain, |
| const CCValAssign &VA, const SDLoc &DL, |
| const LoongArchTargetLowering &TLI) { |
| MachineFunction &MF = DAG.getMachineFunction(); |
| MachineRegisterInfo &RegInfo = MF.getRegInfo(); |
| EVT LocVT = VA.getLocVT(); |
| SDValue Val; |
| const TargetRegisterClass *RC = TLI.getRegClassFor(LocVT.getSimpleVT()); |
| Register VReg = RegInfo.createVirtualRegister(RC); |
| RegInfo.addLiveIn(VA.getLocReg(), VReg); |
| Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); |
| |
| return convertLocVTToValVT(DAG, Val, VA, DL); |
| } |
| |
| // The caller is responsible for loading the full value if the argument is |
| // passed with CCValAssign::Indirect. |
| static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, |
| const CCValAssign &VA, const SDLoc &DL) { |
| MachineFunction &MF = DAG.getMachineFunction(); |
| MachineFrameInfo &MFI = MF.getFrameInfo(); |
| EVT ValVT = VA.getValVT(); |
| int FI = MFI.CreateFixedObject(ValVT.getStoreSize(), VA.getLocMemOffset(), |
| /*IsImmutable=*/true); |
| SDValue FIN = DAG.getFrameIndex( |
| FI, MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0))); |
| |
| ISD::LoadExtType ExtType; |
| switch (VA.getLocInfo()) { |
| default: |
| llvm_unreachable("Unexpected CCValAssign::LocInfo"); |
| case CCValAssign::Full: |
| case CCValAssign::Indirect: |
| case CCValAssign::BCvt: |
| ExtType = ISD::NON_EXTLOAD; |
| break; |
| } |
| return DAG.getExtLoad( |
| ExtType, DL, VA.getLocVT(), Chain, FIN, |
| MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT); |
| } |
| |
| static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, |
| const CCValAssign &VA, const SDLoc &DL) { |
| EVT LocVT = VA.getLocVT(); |
| |
| switch (VA.getLocInfo()) { |
| default: |
| llvm_unreachable("Unexpected CCValAssign::LocInfo"); |
| case CCValAssign::Full: |
| break; |
| case CCValAssign::BCvt: |
| if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) |
| Val = DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, MVT::i64, Val); |
| else |
| Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val); |
| break; |
| } |
| return Val; |
| } |
| |
| static bool CC_LoongArch_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, |
| CCValAssign::LocInfo LocInfo, |
| ISD::ArgFlagsTy ArgFlags, CCState &State) { |
| if (LocVT == MVT::i32 || LocVT == MVT::i64) { |
| // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, SpLim |
| // s0 s1 s2 s3 s4 s5 s6 s7 s8 |
| static const MCPhysReg GPRList[] = { |
| LoongArch::R23, LoongArch::R24, LoongArch::R25, LoongArch::R26, LoongArch::R27, |
| LoongArch::R28, LoongArch::R29, LoongArch::R30, LoongArch::R31}; |
| if (unsigned Reg = State.AllocateReg(GPRList)) { |
| State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); |
| return false; |
| } |
| } |
| |
| if (LocVT == MVT::f32) { |
| // Pass in STG registers: F1, F2, F3, F4 |
| // fs0,fs1,fs2,fs3 |
| static const MCPhysReg FPR32List[] = {LoongArch::F24, LoongArch::F25, |
| LoongArch::F26, LoongArch::F27}; |
| if (unsigned Reg = State.AllocateReg(FPR32List)) { |
| State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); |
| return false; |
| } |
| } |
| |
| if (LocVT == MVT::f64) { |
| // Pass in STG registers: D1, D2, D3, D4 |
| // fs4,fs5,fs6,fs7 |
| static const MCPhysReg FPR64List[] = {LoongArch::F28_64, LoongArch::F29_64, |
| LoongArch::F30_64, LoongArch::F31_64}; |
| if (unsigned Reg = State.AllocateReg(FPR64List)) { |
| State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); |
| return false; |
| } |
| } |
| |
| report_fatal_error("No registers left in GHC calling convention"); |
| return true; |
| } |
| |
| // Transform physical registers into virtual registers. |
| SDValue LoongArchTargetLowering::LowerFormalArguments( |
| SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, |
| const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, |
| SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { |
| |
| MachineFunction &MF = DAG.getMachineFunction(); |
| |
| switch (CallConv) { |
| default: |
| llvm_unreachable("Unsupported calling convention"); |
| case CallingConv::C: |
| case CallingConv::Fast: |
| break; |
| case CallingConv::GHC: |
| if (!MF.getSubtarget().getFeatureBits()[LoongArch::FeatureBasicF] || |
| !MF.getSubtarget().getFeatureBits()[LoongArch::FeatureBasicD]) |
| report_fatal_error( |
| "GHC calling convention requires the F and D extensions"); |
| } |
| |
| EVT PtrVT = getPointerTy(DAG.getDataLayout()); |
| MVT GRLenVT = Subtarget.getGRLenVT(); |
| unsigned GRLenInBytes = Subtarget.getGRLen() / 8; |
| // Used with varargs to acumulate store chains. |
| std::vector<SDValue> OutChains; |
| |
| // Assign locations to all of the incoming arguments. |
| SmallVector<CCValAssign> ArgLocs; |
| CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); |
| |
| if (CallConv == CallingConv::GHC) |
| CCInfo.AnalyzeFormalArguments(Ins, CC_LoongArch_GHC); |
| else |
| analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false, CC_LoongArch); |
| |
| for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { |
| CCValAssign &VA = ArgLocs[i]; |
| SDValue ArgValue; |
| if (VA.isRegLoc()) |
| ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, *this); |
| else |
| ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL); |
| if (VA.getLocInfo() == CCValAssign::Indirect) { |
| // If the original argument was split and passed by reference, we need to |
| // load all parts of it here (using the same address). |
| InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue, |
| MachinePointerInfo())); |
| unsigned ArgIndex = Ins[i].OrigArgIndex; |
| unsigned ArgPartOffset = Ins[i].PartOffset; |
| assert(ArgPartOffset == 0); |
| while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) { |
| CCValAssign &PartVA = ArgLocs[i + 1]; |
| unsigned PartOffset = Ins[i + 1].PartOffset - ArgPartOffset; |
| SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL); |
| SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset); |
| InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address, |
| MachinePointerInfo())); |
| ++i; |
| } |
| continue; |
| } |
| InVals.push_back(ArgValue); |
| } |
| |
| if (IsVarArg) { |
| ArrayRef<MCPhysReg> ArgRegs = ArrayRef(ArgGPRs); |
| unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs); |
| const TargetRegisterClass *RC = &LoongArch::GPRRegClass; |
| MachineFrameInfo &MFI = MF.getFrameInfo(); |
| MachineRegisterInfo &RegInfo = MF.getRegInfo(); |
| auto *LoongArchFI = MF.getInfo<LoongArchMachineFunctionInfo>(); |
| |
| // Offset of the first variable argument from stack pointer, and size of |
| // the vararg save area. For now, the varargs save area is either zero or |
| // large enough to hold a0-a7. |
| int VaArgOffset, VarArgsSaveSize; |
| |
| // If all registers are allocated, then all varargs must be passed on the |
| // stack and we don't need to save any argregs. |
| if (ArgRegs.size() == Idx) { |
| VaArgOffset = CCInfo.getNextStackOffset(); |
| VarArgsSaveSize = 0; |
| } else { |
| VarArgsSaveSize = GRLenInBytes * (ArgRegs.size() - Idx); |
| VaArgOffset = -VarArgsSaveSize; |
| } |
| |
| // Record the frame index of the first variable argument |
| // which is a value necessary to VASTART. |
| int FI = MFI.CreateFixedObject(GRLenInBytes, VaArgOffset, true); |
| LoongArchFI->setVarArgsFrameIndex(FI); |
| |
| // If saving an odd number of registers then create an extra stack slot to |
| // ensure that the frame pointer is 2*GRLen-aligned, which in turn ensures |
| // offsets to even-numbered registered remain 2*GRLen-aligned. |
| if (Idx % 2) { |
| MFI.CreateFixedObject(GRLenInBytes, VaArgOffset - (int)GRLenInBytes, |
| true); |
| VarArgsSaveSize += GRLenInBytes; |
| } |
| |
| // Copy the integer registers that may have been used for passing varargs |
| // to the vararg save area. |
| for (unsigned I = Idx; I < ArgRegs.size(); |
| ++I, VaArgOffset += GRLenInBytes) { |
| const Register Reg = RegInfo.createVirtualRegister(RC); |
| RegInfo.addLiveIn(ArgRegs[I], Reg); |
| SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, GRLenVT); |
| FI = MFI.CreateFixedObject(GRLenInBytes, VaArgOffset, true); |
| SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout())); |
| SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff, |
| MachinePointerInfo::getFixedStack(MF, FI)); |
| cast<StoreSDNode>(Store.getNode()) |
| ->getMemOperand() |
| ->setValue((Value *)nullptr); |
| OutChains.push_back(Store); |
| } |
| LoongArchFI->setVarArgsSaveSize(VarArgsSaveSize); |
| } |
| |
| // All stores are grouped in one node to allow the matching between |
| // the size of Ins and InVals. This only happens for vararg functions. |
| if (!OutChains.empty()) { |
| OutChains.push_back(Chain); |
| Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains); |
| } |
| |
| return Chain; |
| } |
| |
| bool LoongArchTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { |
| return CI->isTailCall(); |
| } |
| |
| // Check whether the call is eligible for tail call optimization. |
| bool LoongArchTargetLowering::isEligibleForTailCallOptimization( |
| CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF, |
| const SmallVectorImpl<CCValAssign> &ArgLocs) const { |
| |
| auto CalleeCC = CLI.CallConv; |
| auto &Outs = CLI.Outs; |
| auto &Caller = MF.getFunction(); |
| auto CallerCC = Caller.getCallingConv(); |
| |
| // Do not tail call opt if the stack is used to pass parameters. |
| if (CCInfo.getNextStackOffset() != 0) |
| return false; |
| |
| // Do not tail call opt if any parameters need to be passed indirectly. |
| for (auto &VA : ArgLocs) |
| if (VA.getLocInfo() == CCValAssign::Indirect) |
| return false; |
| |
| // Do not tail call opt if either caller or callee uses struct return |
| // semantics. |
| auto IsCallerStructRet = Caller.hasStructRetAttr(); |
| auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet(); |
| if (IsCallerStructRet || IsCalleeStructRet) |
| return false; |
| |
| // Do not tail call opt if either the callee or caller has a byval argument. |
| for (auto &Arg : Outs) |
| if (Arg.Flags.isByVal()) |
| return false; |
| |
| // The callee has to preserve all registers the caller needs to preserve. |
| const LoongArchRegisterInfo *TRI = Subtarget.getRegisterInfo(); |
| const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC); |
| if (CalleeCC != CallerCC) { |
| const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC); |
| if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) |
| return false; |
| } |
| return true; |
| } |
| |
| static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) { |
| return DAG.getDataLayout().getPrefTypeAlign( |
| VT.getTypeForEVT(*DAG.getContext())); |
| } |
| |
| // Lower a call to a callseq_start + CALL + callseq_end chain, and add input |
| // and output parameter nodes. |
| SDValue |
| LoongArchTargetLowering::LowerCall(CallLoweringInfo &CLI, |
| SmallVectorImpl<SDValue> &InVals) const { |
| SelectionDAG &DAG = CLI.DAG; |
| SDLoc &DL = CLI.DL; |
| SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; |
| SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; |
| SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; |
| SDValue Chain = CLI.Chain; |
| SDValue Callee = CLI.Callee; |
| CallingConv::ID CallConv = CLI.CallConv; |
| bool IsVarArg = CLI.IsVarArg; |
| EVT PtrVT = getPointerTy(DAG.getDataLayout()); |
| MVT GRLenVT = Subtarget.getGRLenVT(); |
| bool &IsTailCall = CLI.IsTailCall; |
| |
| MachineFunction &MF = DAG.getMachineFunction(); |
| |
| // Analyze the operands of the call, assigning locations to each operand. |
| SmallVector<CCValAssign> ArgLocs; |
| CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); |
| |
| if (CallConv == CallingConv::GHC) |
| ArgCCInfo.AnalyzeCallOperands(Outs, CC_LoongArch_GHC); |
| else |
| analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI, CC_LoongArch); |
| |
| // Check if it's really possible to do a tail call. |
| if (IsTailCall) |
| IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs); |
| |
| if (IsTailCall) |
| ++NumTailCalls; |
| else if (CLI.CB && CLI.CB->isMustTailCall()) |
| report_fatal_error("failed to perform tail call elimination on a call " |
| "site marked musttail"); |
| |
| // Get a count of how many bytes are to be pushed on the stack. |
| unsigned NumBytes = ArgCCInfo.getNextStackOffset(); |
| |
| // Create local copies for byval args. |
| SmallVector<SDValue> ByValArgs; |
| for (unsigned i = 0, e = Outs.size(); i != e; ++i) { |
| ISD::ArgFlagsTy Flags = Outs[i].Flags; |
| if (!Flags.isByVal()) |
| continue; |
| |
| SDValue Arg = OutVals[i]; |
| unsigned Size = Flags.getByValSize(); |
| Align Alignment = Flags.getNonZeroByValAlign(); |
| |
| int FI = |
| MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/false); |
| SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout())); |
| SDValue SizeNode = DAG.getConstant(Size, DL, GRLenVT); |
| |
| Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment, |
| /*IsVolatile=*/false, |
| /*AlwaysInline=*/false, /*isTailCall=*/IsTailCall, |
| MachinePointerInfo(), MachinePointerInfo()); |
| ByValArgs.push_back(FIPtr); |
| } |
| |
| if (!IsTailCall) |
| Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL); |
| |
| // Copy argument values to their designated locations. |
| SmallVector<std::pair<Register, SDValue>> RegsToPass; |
| SmallVector<SDValue> MemOpChains; |
| SDValue StackPtr; |
| for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) { |
| CCValAssign &VA = ArgLocs[i]; |
| SDValue ArgValue = OutVals[i]; |
| ISD::ArgFlagsTy Flags = Outs[i].Flags; |
| |
| // Promote the value if needed. |
| // For now, only handle fully promoted and indirect arguments. |
| if (VA.getLocInfo() == CCValAssign::Indirect) { |
| // Store the argument in a stack slot and pass its address. |
| Align StackAlign = |
| std::max(getPrefTypeAlign(Outs[i].ArgVT, DAG), |
| getPrefTypeAlign(ArgValue.getValueType(), DAG)); |
| TypeSize StoredSize = ArgValue.getValueType().getStoreSize(); |
| // If the original argument was split and passed by reference, we need to |
| // store the required parts of it here (and pass just one address). |
| unsigned ArgIndex = Outs[i].OrigArgIndex; |
| unsigned ArgPartOffset = Outs[i].PartOffset; |
| assert(ArgPartOffset == 0); |
| // Calculate the total size to store. We don't have access to what we're |
| // actually storing other than performing the loop and collecting the |
| // info. |
| SmallVector<std::pair<SDValue, SDValue>> Parts; |
| while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) { |
| SDValue PartValue = OutVals[i + 1]; |
| unsigned PartOffset = Outs[i + 1].PartOffset - ArgPartOffset; |
| SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL); |
| EVT PartVT = PartValue.getValueType(); |
| |
| StoredSize += PartVT.getStoreSize(); |
| StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG)); |
| Parts.push_back(std::make_pair(PartValue, Offset)); |
| ++i; |
| } |
| SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign); |
| int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); |
| MemOpChains.push_back( |
| DAG.getStore(Chain, DL, ArgValue, SpillSlot, |
| MachinePointerInfo::getFixedStack(MF, FI))); |
| for (const auto &Part : Parts) { |
| SDValue PartValue = Part.first; |
| SDValue PartOffset = Part.second; |
| SDValue Address = |
| DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset); |
| MemOpChains.push_back( |
| DAG.getStore(Chain, DL, PartValue, Address, |
| MachinePointerInfo::getFixedStack(MF, FI))); |
| } |
| ArgValue = SpillSlot; |
| } else { |
| ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL); |
| } |
| |
| // Use local copy if it is a byval arg. |
| if (Flags.isByVal()) |
| ArgValue = ByValArgs[j++]; |
| |
| if (VA.isRegLoc()) { |
| // Queue up the argument copies and emit them at the end. |
| RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue)); |
| } else { |
| assert(VA.isMemLoc() && "Argument not register or memory"); |
| assert(!IsTailCall && "Tail call not allowed if stack is used " |
| "for passing parameters"); |
| |
| // Work out the address of the stack slot. |
| if (!StackPtr.getNode()) |
| StackPtr = DAG.getCopyFromReg(Chain, DL, LoongArch::R3, PtrVT); |
| SDValue Address = |
| DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, |
| DAG.getIntPtrConstant(VA.getLocMemOffset(), DL)); |
| |
| // Emit the store. |
| MemOpChains.push_back( |
| DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo())); |
| } |
| } |
| |
| // Join the stores, which are independent of one another. |
| if (!MemOpChains.empty()) |
| Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); |
| |
| SDValue Glue; |
| |
| // Build a sequence of copy-to-reg nodes, chained and glued together. |
| for (auto &Reg : RegsToPass) { |
| Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue); |
| Glue = Chain.getValue(1); |
| } |
| |
| // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a |
| // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't |
| // split it and then direct call can be matched by PseudoCALL. |
| if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) { |
| const GlobalValue *GV = S->getGlobal(); |
| unsigned OpFlags = |
| getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV) |
| ? LoongArchII::MO_CALL |
| : LoongArchII::MO_CALL_PLT; |
| Callee = DAG.getTargetGlobalAddress(S->getGlobal(), DL, PtrVT, 0, OpFlags); |
| } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) { |
| unsigned OpFlags = getTargetMachine().shouldAssumeDSOLocal( |
| *MF.getFunction().getParent(), nullptr) |
| ? LoongArchII::MO_CALL |
| : LoongArchII::MO_CALL_PLT; |
| Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags); |
| } |
| |
| // The first call operand is the chain and the second is the target address. |
| SmallVector<SDValue> Ops; |
| Ops.push_back(Chain); |
| Ops.push_back(Callee); |
| |
| // Add argument registers to the end of the list so that they are |
| // known live into the call. |
| for (auto &Reg : RegsToPass) |
| Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType())); |
| |
| if (!IsTailCall) { |
| // Add a register mask operand representing the call-preserved registers. |
| const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); |
| const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv); |
| assert(Mask && "Missing call preserved mask for calling convention"); |
| Ops.push_back(DAG.getRegisterMask(Mask)); |
| } |
| |
| // Glue the call to the argument copies, if any. |
| if (Glue.getNode()) |
| Ops.push_back(Glue); |
| |
| // Emit the call. |
| SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); |
| |
| if (IsTailCall) { |
| MF.getFrameInfo().setHasTailCall(); |
| return DAG.getNode(LoongArchISD::TAIL, DL, NodeTys, Ops); |
| } |
| |
| Chain = DAG.getNode(LoongArchISD::CALL, DL, NodeTys, Ops); |
| DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); |
| Glue = Chain.getValue(1); |
| |
| // Mark the end of the call, which is glued to the call itself. |
| Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, Glue, DL); |
| Glue = Chain.getValue(1); |
| |
| // Assign locations to each value returned by this call. |
| SmallVector<CCValAssign> RVLocs; |
| CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext()); |
| analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true, CC_LoongArch); |
| |
| // Copy all of the result registers out of their specified physreg. |
| for (auto &VA : RVLocs) { |
| // Copy the value out. |
| SDValue RetValue = |
| DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue); |
| // Glue the RetValue to the end of the call sequence. |
| Chain = RetValue.getValue(1); |
| Glue = RetValue.getValue(2); |
| |
| RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL); |
| |
| InVals.push_back(RetValue); |
| } |
| |
| return Chain; |
| } |
| |
| bool LoongArchTargetLowering::CanLowerReturn( |
| CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, |
| const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const { |
| SmallVector<CCValAssign> RVLocs; |
| CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context); |
| |
| for (unsigned i = 0, e = Outs.size(); i != e; ++i) { |
| LoongArchABI::ABI ABI = |
| MF.getSubtarget<LoongArchSubtarget>().getTargetABI(); |
| if (CC_LoongArch(MF.getDataLayout(), ABI, i, Outs[i].VT, CCValAssign::Full, |
| Outs[i].Flags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, |
| nullptr)) |
| return false; |
| } |
| return true; |
| } |
| |
| SDValue LoongArchTargetLowering::LowerReturn( |
| SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, |
| const SmallVectorImpl<ISD::OutputArg> &Outs, |
| const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL, |
| SelectionDAG &DAG) const { |
| // Stores the assignment of the return value to a location. |
| SmallVector<CCValAssign> RVLocs; |
| |
| // Info about the registers and stack slot. |
| CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs, |
| *DAG.getContext()); |
| |
| analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true, |
| nullptr, CC_LoongArch); |
| if (CallConv == CallingConv::GHC && !RVLocs.empty()) |
| report_fatal_error("GHC functions return void only"); |
| SDValue Glue; |
| SmallVector<SDValue, 4> RetOps(1, Chain); |
| |
| // Copy the result values into the output registers. |
| for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) { |
| CCValAssign &VA = RVLocs[i]; |
| assert(VA.isRegLoc() && "Can only return in registers!"); |
| |
| // Handle a 'normal' return. |
| SDValue Val = convertValVTToLocVT(DAG, OutVals[i], VA, DL); |
| Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue); |
| |
| // Guarantee that all emitted copies are stuck together. |
| Glue = Chain.getValue(1); |
| RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); |
| } |
| |
| RetOps[0] = Chain; // Update chain. |
| |
| // Add the glue node if we have it. |
| if (Glue.getNode()) |
| RetOps.push_back(Glue); |
| |
| return DAG.getNode(LoongArchISD::RET, DL, MVT::Other, RetOps); |
| } |
| |
| bool LoongArchTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, |
| bool ForCodeSize) const { |
| // TODO: Maybe need more checks here after vector extension is supported. |
| if (VT == MVT::f32 && !Subtarget.hasBasicF()) |
| return false; |
| if (VT == MVT::f64 && !Subtarget.hasBasicD()) |
| return false; |
| return (Imm.isZero() || Imm.isExactlyValue(+1.0)); |
| } |
| |
| bool LoongArchTargetLowering::isCheapToSpeculateCttz(Type *) const { |
| return true; |
| } |
| |
| bool LoongArchTargetLowering::isCheapToSpeculateCtlz(Type *) const { |
| return true; |
| } |
| |
| bool LoongArchTargetLowering::shouldInsertFencesForAtomic( |
| const Instruction *I) const { |
| if (!Subtarget.is64Bit()) |
| return isa<LoadInst>(I) || isa<StoreInst>(I); |
| |
| if (isa<LoadInst>(I)) |
| return true; |
| |
| // On LA64, atomic store operations with IntegerBitWidth of 32 and 64 do not |
| // require fences beacuse we can use amswap_db.[w/d]. |
| if (isa<StoreInst>(I)) { |
| unsigned Size = I->getOperand(0)->getType()->getIntegerBitWidth(); |
| return (Size == 8 || Size == 16); |
| } |
| |
| return false; |
| } |
| |
| EVT LoongArchTargetLowering::getSetCCResultType(const DataLayout &DL, |
| LLVMContext &Context, |
| EVT VT) const { |
| if (!VT.isVector()) |
| return getPointerTy(DL); |
| return VT.changeVectorElementTypeToInteger(); |
| } |
| |
| bool LoongArchTargetLowering::hasAndNot(SDValue Y) const { |
| // TODO: Support vectors. |
| return Y.getValueType().isScalarInteger() && !isa<ConstantSDNode>(Y); |
| } |
| |
| bool LoongArchTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, |
| const CallInst &I, |
| MachineFunction &MF, |
| unsigned Intrinsic) const { |
| switch (Intrinsic) { |
| default: |
| return false; |
| case Intrinsic::loongarch_masked_atomicrmw_xchg_i32: |
| case Intrinsic::loongarch_masked_atomicrmw_add_i32: |
| case Intrinsic::loongarch_masked_atomicrmw_sub_i32: |
| case Intrinsic::loongarch_masked_atomicrmw_nand_i32: |
| Info.opc = ISD::INTRINSIC_W_CHAIN; |
| Info.memVT = MVT::i32; |
| Info.ptrVal = I.getArgOperand(0); |
| Info.offset = 0; |
| Info.align = Align(4); |
| Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore | |
| MachineMemOperand::MOVolatile; |
| return true; |
| // TODO: Add more Intrinsics later. |
| } |
| } |
| |
| TargetLowering::AtomicExpansionKind |
| LoongArchTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { |
| // TODO: Add more AtomicRMWInst that needs to be extended. |
| |
| // Since floating-point operation requires a non-trivial set of data |
| // operations, use CmpXChg to expand. |
| if (AI->isFloatingPointOperation() || |
| AI->getOperation() == AtomicRMWInst::UIncWrap || |
| AI->getOperation() == AtomicRMWInst::UDecWrap) |
| return AtomicExpansionKind::CmpXChg; |
| |
| unsigned Size = AI->getType()->getPrimitiveSizeInBits(); |
| if (Size == 8 || Size == 16) |
| return AtomicExpansionKind::MaskedIntrinsic; |
| return AtomicExpansionKind::None; |
| } |
| |
| static Intrinsic::ID |
| getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, |
| AtomicRMWInst::BinOp BinOp) { |
| if (GRLen == 64) { |
| switch (BinOp) { |
| default: |
| llvm_unreachable("Unexpected AtomicRMW BinOp"); |
| case AtomicRMWInst::Xchg: |
| return Intrinsic::loongarch_masked_atomicrmw_xchg_i64; |
| case AtomicRMWInst::Add: |
| return Intrinsic::loongarch_masked_atomicrmw_add_i64; |
| case AtomicRMWInst::Sub: |
| return Intrinsic::loongarch_masked_atomicrmw_sub_i64; |
| case AtomicRMWInst::Nand: |
| return Intrinsic::loongarch_masked_atomicrmw_nand_i64; |
| case AtomicRMWInst::UMax: |
| return Intrinsic::loongarch_masked_atomicrmw_umax_i64; |
| case AtomicRMWInst::UMin: |
| return Intrinsic::loongarch_masked_atomicrmw_umin_i64; |
| case AtomicRMWInst::Max: |
| return Intrinsic::loongarch_masked_atomicrmw_max_i64; |
| case AtomicRMWInst::Min: |
| return Intrinsic::loongarch_masked_atomicrmw_min_i64; |
| // TODO: support other AtomicRMWInst. |
| } |
| } |
| |
| if (GRLen == 32) { |
| switch (BinOp) { |
| default: |
| llvm_unreachable("Unexpected AtomicRMW BinOp"); |
| case AtomicRMWInst::Xchg: |
| return Intrinsic::loongarch_masked_atomicrmw_xchg_i32; |
| case AtomicRMWInst::Add: |
| return Intrinsic::loongarch_masked_atomicrmw_add_i32; |
| case AtomicRMWInst::Sub: |
| return Intrinsic::loongarch_masked_atomicrmw_sub_i32; |
| case AtomicRMWInst::Nand: |
| return Intrinsic::loongarch_masked_atomicrmw_nand_i32; |
| // TODO: support other AtomicRMWInst. |
| } |
| } |
| |
| llvm_unreachable("Unexpected GRLen\n"); |
| } |
| |
| TargetLowering::AtomicExpansionKind |
| LoongArchTargetLowering::shouldExpandAtomicCmpXchgInIR( |
| AtomicCmpXchgInst *CI) const { |
| unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits(); |
| if (Size == 8 || Size == 16) |
| return AtomicExpansionKind::MaskedIntrinsic; |
| return AtomicExpansionKind::None; |
| } |
| |
| Value *LoongArchTargetLowering::emitMaskedAtomicCmpXchgIntrinsic( |
| IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, |
| Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const { |
| Value *Ordering = |
| Builder.getIntN(Subtarget.getGRLen(), static_cast<uint64_t>(Ord)); |
| |
| // TODO: Support cmpxchg on LA32. |
| Intrinsic::ID CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i64; |
| CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty()); |
| NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty()); |
| Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty()); |
| Type *Tys[] = {AlignedAddr->getType()}; |
| Function *MaskedCmpXchg = |
| Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys); |
| Value *Result = Builder.CreateCall( |
| MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering}); |
| Result = Builder.CreateTrunc(Result, Builder.getInt32Ty()); |
| return Result; |
| } |
| |
| Value *LoongArchTargetLowering::emitMaskedAtomicRMWIntrinsic( |
| IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, |
| Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const { |
| unsigned GRLen = Subtarget.getGRLen(); |
| Value *Ordering = |
| Builder.getIntN(GRLen, static_cast<uint64_t>(AI->getOrdering())); |
| Type *Tys[] = {AlignedAddr->getType()}; |
| Function *LlwOpScwLoop = Intrinsic::getDeclaration( |
| AI->getModule(), |
| getIntrinsicForMaskedAtomicRMWBinOp(GRLen, AI->getOperation()), Tys); |
| |
| if (GRLen == 64) { |
| Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty()); |
| Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty()); |
| ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty()); |
| } |
| |
| Value *Result; |
| |
| // Must pass the shift amount needed to sign extend the loaded value prior |
| // to performing a signed comparison for min/max. ShiftAmt is the number of |
| // bits to shift the value into position. Pass GRLen-ShiftAmt-ValWidth, which |
| // is the number of bits to left+right shift the value in order to |
| // sign-extend. |
| if (AI->getOperation() == AtomicRMWInst::Min || |
| AI->getOperation() == AtomicRMWInst::Max) { |
| const DataLayout &DL = AI->getModule()->getDataLayout(); |
| unsigned ValWidth = |
| DL.getTypeStoreSizeInBits(AI->getValOperand()->getType()); |
| Value *SextShamt = |
| Builder.CreateSub(Builder.getIntN(GRLen, GRLen - ValWidth), ShiftAmt); |
| Result = Builder.CreateCall(LlwOpScwLoop, |
| {AlignedAddr, Incr, Mask, SextShamt, Ordering}); |
| } else { |
| Result = |
| Builder.CreateCall(LlwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering}); |
| } |
| |
| if (GRLen == 64) |
| Result = Builder.CreateTrunc(Result, Builder.getInt32Ty()); |
| return Result; |
| } |
| |
| bool LoongArchTargetLowering::isFMAFasterThanFMulAndFAdd( |
| const MachineFunction &MF, EVT VT) const { |
| VT = VT.getScalarType(); |
| |
| if (!VT.isSimple()) |
| return false; |
| |
| switch (VT.getSimpleVT().SimpleTy) { |
| case MVT::f32: |
| case MVT::f64: |
| return true; |
| default: |
| break; |
| } |
| |
| return false; |
| } |
| |
| Register LoongArchTargetLowering::getExceptionPointerRegister( |
| const Constant *PersonalityFn) const { |
| return LoongArch::R4; |
| } |
| |
| Register LoongArchTargetLowering::getExceptionSelectorRegister( |
| const Constant *PersonalityFn) const { |
| return LoongArch::R5; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // LoongArch Inline Assembly Support |
| //===----------------------------------------------------------------------===// |
| |
| LoongArchTargetLowering::ConstraintType |
| LoongArchTargetLowering::getConstraintType(StringRef Constraint) const { |
| // LoongArch specific constraints in GCC: config/loongarch/constraints.md |
| // |
| // 'f': A floating-point register (if available). |
| // 'k': A memory operand whose address is formed by a base register and |
| // (optionally scaled) index register. |
| // 'l': A signed 16-bit constant. |
| // 'm': A memory operand whose address is formed by a base register and |
| // offset that is suitable for use in instructions with the same |
| // addressing mode as st.w and ld.w. |
| // 'I': A signed 12-bit constant (for arithmetic instructions). |
| // 'J': Integer zero. |
| // 'K': An unsigned 12-bit constant (for logic instructions). |
| // "ZB": An address that is held in a general-purpose register. The offset is |
| // zero. |
| // "ZC": A memory operand whose address is formed by a base register and |
| // offset that is suitable for use in instructions with the same |
| // addressing mode as ll.w and sc.w. |
| if (Constraint.size() == 1) { |
| switch (Constraint[0]) { |
| default: |
| break; |
| case 'f': |
| return C_RegisterClass; |
| case 'l': |
| case 'I': |
| case 'J': |
| case 'K': |
| return C_Immediate; |
| case 'k': |
| return C_Memory; |
| } |
| } |
| |
| if (Constraint == "ZC" || Constraint == "ZB") |
| return C_Memory; |
| |
| // 'm' is handled here. |
| return TargetLowering::getConstraintType(Constraint); |
| } |
| |
| unsigned LoongArchTargetLowering::getInlineAsmMemConstraint( |
| StringRef ConstraintCode) const { |
| return StringSwitch<unsigned>(ConstraintCode) |
| .Case("k", InlineAsm::Constraint_k) |
| .Case("ZB", InlineAsm::Constraint_ZB) |
| .Case("ZC", InlineAsm::Constraint_ZC) |
| .Default(TargetLowering::getInlineAsmMemConstraint(ConstraintCode)); |
| } |
| |
| std::pair<unsigned, const TargetRegisterClass *> |
| LoongArchTargetLowering::getRegForInlineAsmConstraint( |
| const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { |
| // First, see if this is a constraint that directly corresponds to a LoongArch |
| // register class. |
| if (Constraint.size() == 1) { |
| switch (Constraint[0]) { |
| case 'r': |
| // TODO: Support fixed vectors up to GRLen? |
| if (VT.isVector()) |
| break; |
| return std::make_pair(0U, &LoongArch::GPRRegClass); |
| case 'f': |
| if (Subtarget.hasBasicF() && VT == MVT::f32) |
| return std::make_pair(0U, &LoongArch::FPR32RegClass); |
| if (Subtarget.hasBasicD() && VT == MVT::f64) |
| return std::make_pair(0U, &LoongArch::FPR64RegClass); |
| break; |
| default: |
| break; |
| } |
| } |
| |
| // TargetLowering::getRegForInlineAsmConstraint uses the name of the TableGen |
| // record (e.g. the "R0" in `def R0`) to choose registers for InlineAsm |
| // constraints while the official register name is prefixed with a '$'. So we |
| // clip the '$' from the original constraint string (e.g. {$r0} to {r0}.) |
| // before it being parsed. And TargetLowering::getRegForInlineAsmConstraint is |
| // case insensitive, so no need to convert the constraint to upper case here. |
| // |
| // For now, no need to support ABI names (e.g. `$a0`) as clang will correctly |
| // decode the usage of register name aliases into their official names. And |
| // AFAIK, the not yet upstreamed `rustc` for LoongArch will always use |
| // official register names. |
| if (Constraint.startswith("{$r") || Constraint.startswith("{$f")) { |
| bool IsFP = Constraint[2] == 'f'; |
| std::pair<StringRef, StringRef> Temp = Constraint.split('$'); |
| std::pair<unsigned, const TargetRegisterClass *> R; |
| R = TargetLowering::getRegForInlineAsmConstraint( |
| TRI, join_items("", Temp.first, Temp.second), VT); |
| // Match those names to the widest floating point register type available. |
| if (IsFP) { |
| unsigned RegNo = R.first; |
| if (LoongArch::F0 <= RegNo && RegNo <= LoongArch::F31) { |
| if (Subtarget.hasBasicD() && (VT == MVT::f64 || VT == MVT::Other)) { |
| unsigned DReg = RegNo - LoongArch::F0 + LoongArch::F0_64; |
| return std::make_pair(DReg, &LoongArch::FPR64RegClass); |
| } |
| } |
| } |
| return R; |
| } |
| |
| return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); |
| } |
| |
| void LoongArchTargetLowering::LowerAsmOperandForConstraint( |
| SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops, |
| SelectionDAG &DAG) const { |
| // Currently only support length 1 constraints. |
| if (Constraint.length() == 1) { |
| switch (Constraint[0]) { |
| case 'l': |
| // Validate & create a 16-bit signed immediate operand. |
| if (auto *C = dyn_cast<ConstantSDNode>(Op)) { |
| uint64_t CVal = C->getSExtValue(); |
| if (isInt<16>(CVal)) |
| Ops.push_back( |
| DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getGRLenVT())); |
| } |
| return; |
| case 'I': |
| // Validate & create a 12-bit signed immediate operand. |
| if (auto *C = dyn_cast<ConstantSDNode>(Op)) { |
| uint64_t CVal = C->getSExtValue(); |
| if (isInt<12>(CVal)) |
| Ops.push_back( |
| DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getGRLenVT())); |
| } |
| return; |
| case 'J': |
| // Validate & create an integer zero operand. |
| if (auto *C = dyn_cast<ConstantSDNode>(Op)) |
| if (C->getZExtValue() == 0) |
| Ops.push_back( |
| DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getGRLenVT())); |
| return; |
| case 'K': |
| // Validate & create a 12-bit unsigned immediate operand. |
| if (auto *C = dyn_cast<ConstantSDNode>(Op)) { |
| uint64_t CVal = C->getZExtValue(); |
| if (isUInt<12>(CVal)) |
| Ops.push_back( |
| DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getGRLenVT())); |
| } |
| return; |
| default: |
| break; |
| } |
| } |
| TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); |
| } |
| |
| #define GET_REGISTER_MATCHER |
| #include "LoongArchGenAsmMatcher.inc" |
| |
| Register |
| LoongArchTargetLowering::getRegisterByName(const char *RegName, LLT VT, |
| const MachineFunction &MF) const { |
| std::pair<StringRef, StringRef> Name = StringRef(RegName).split('$'); |
| std::string NewRegName = Name.second.str(); |
| Register Reg = MatchRegisterAltName(NewRegName); |
| if (Reg == LoongArch::NoRegister) |
| Reg = MatchRegisterName(NewRegName); |
| if (Reg == LoongArch::NoRegister) |
| report_fatal_error( |
| Twine("Invalid register name \"" + StringRef(RegName) + "\".")); |
| BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF); |
| if (!ReservedRegs.test(Reg)) |
| report_fatal_error(Twine("Trying to obtain non-reserved register \"" + |
| StringRef(RegName) + "\".")); |
| return Reg; |
| } |
| |
| bool LoongArchTargetLowering::decomposeMulByConstant(LLVMContext &Context, |
| EVT VT, SDValue C) const { |
| // TODO: Support vectors. |
| if (!VT.isScalarInteger()) |
| return false; |
| |
| // Omit the optimization if the data size exceeds GRLen. |
| if (VT.getSizeInBits() > Subtarget.getGRLen()) |
| return false; |
| |
| // Break MUL into (SLLI + ADD/SUB) or ALSL. |
| if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) { |
| const APInt &Imm = ConstNode->getAPIntValue(); |
| if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() || |
| (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2()) |
| return true; |
| } |
| |
| return false; |
| } |