| //=== lib/CodeGen/GlobalISel/AMDGPUCombinerHelper.cpp ---------------------===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "AMDGPUCombinerHelper.h" | |
| #include "GCNSubtarget.h" | |
| #include "llvm/CodeGen/GlobalISel/GISelValueTracking.h" | |
| #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" | |
| #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" | |
| #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" | |
| #include "llvm/IR/IntrinsicsAMDGPU.h" | |
| #include "llvm/Target/TargetMachine.h" | |
| using namespace llvm; | |
| using namespace MIPatternMatch; | |
| AMDGPUCombinerHelper::AMDGPUCombinerHelper( | |
| GISelChangeObserver &Observer, MachineIRBuilder &B, bool IsPreLegalize, | |
| GISelValueTracking *VT, MachineDominatorTree *MDT, const LegalizerInfo *LI, | |
| const GCNSubtarget &STI) | |
| : CombinerHelper(Observer, B, IsPreLegalize, VT, MDT, LI), STI(STI), | |
| TII(*STI.getInstrInfo()) {} | |
| LLVM_READNONE | |
| static bool fnegFoldsIntoMI(const MachineInstr &MI) { | |
| switch (MI.getOpcode()) { | |
| case AMDGPU::G_FADD: | |
| case AMDGPU::G_FSUB: | |
| case AMDGPU::G_FMUL: | |
| case AMDGPU::G_FMA: | |
| case AMDGPU::G_FMAD: | |
| case AMDGPU::G_FMINNUM: | |
| case AMDGPU::G_FMAXNUM: | |
| case AMDGPU::G_FMINNUM_IEEE: | |
| case AMDGPU::G_FMAXNUM_IEEE: | |
| case AMDGPU::G_FMINIMUM: | |
| case AMDGPU::G_FMAXIMUM: | |
| case AMDGPU::G_FSIN: | |
| case AMDGPU::G_FPEXT: | |
| case AMDGPU::G_INTRINSIC_TRUNC: | |
| case AMDGPU::G_FPTRUNC: | |
| case AMDGPU::G_FRINT: | |
| case AMDGPU::G_FNEARBYINT: | |
| case AMDGPU::G_INTRINSIC_ROUND: | |
| case AMDGPU::G_INTRINSIC_ROUNDEVEN: | |
| case AMDGPU::G_FCANONICALIZE: | |
| case AMDGPU::G_AMDGPU_RCP_IFLAG: | |
| case AMDGPU::G_AMDGPU_FMIN_LEGACY: | |
| case AMDGPU::G_AMDGPU_FMAX_LEGACY: | |
| return true; | |
| case AMDGPU::G_INTRINSIC: { | |
| Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID(); | |
| switch (IntrinsicID) { | |
| case Intrinsic::amdgcn_rcp: | |
| case Intrinsic::amdgcn_rcp_legacy: | |
| case Intrinsic::amdgcn_sin: | |
| case Intrinsic::amdgcn_fmul_legacy: | |
| case Intrinsic::amdgcn_fmed3: | |
| case Intrinsic::amdgcn_fma_legacy: | |
| return true; | |
| default: | |
| return false; | |
| } | |
| } | |
| default: | |
| return false; | |
| } | |
| } | |
| /// \p returns true if the operation will definitely need to use a 64-bit | |
| /// encoding, and thus will use a VOP3 encoding regardless of the source | |
| /// modifiers. | |
| LLVM_READONLY | |
| static bool opMustUseVOP3Encoding(const MachineInstr &MI, | |
| const MachineRegisterInfo &MRI) { | |
| return MI.getNumOperands() > (isa<GIntrinsic>(MI) ? 4u : 3u) || | |
| MRI.getType(MI.getOperand(0).getReg()).getScalarSizeInBits() == 64; | |
| } | |
| // Most FP instructions support source modifiers. | |
| LLVM_READONLY | |
| static bool hasSourceMods(const MachineInstr &MI) { | |
| if (!MI.memoperands().empty()) | |
| return false; | |
| switch (MI.getOpcode()) { | |
| case AMDGPU::COPY: | |
| case AMDGPU::G_SELECT: | |
| case AMDGPU::G_FDIV: | |
| case AMDGPU::G_FREM: | |
| case TargetOpcode::INLINEASM: | |
| case TargetOpcode::INLINEASM_BR: | |
| case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: | |
| case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: | |
| case AMDGPU::G_BITCAST: | |
| case AMDGPU::G_ANYEXT: | |
| case AMDGPU::G_BUILD_VECTOR: | |
| case AMDGPU::G_BUILD_VECTOR_TRUNC: | |
| case AMDGPU::G_PHI: | |
| return false; | |
| case AMDGPU::G_INTRINSIC: | |
| case AMDGPU::G_INTRINSIC_CONVERGENT: { | |
| Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID(); | |
| switch (IntrinsicID) { | |
| case Intrinsic::amdgcn_interp_p1: | |
| case Intrinsic::amdgcn_interp_p2: | |
| case Intrinsic::amdgcn_interp_mov: | |
| case Intrinsic::amdgcn_interp_p1_f16: | |
| case Intrinsic::amdgcn_interp_p2_f16: | |
| case Intrinsic::amdgcn_div_scale: | |
| return false; | |
| default: | |
| return true; | |
| } | |
| } | |
| default: | |
| return true; | |
| } | |
| } | |
| static bool allUsesHaveSourceMods(MachineInstr &MI, MachineRegisterInfo &MRI, | |
| unsigned CostThreshold = 4) { | |
| // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus | |
| // it is truly free to use a source modifier in all cases. If there are | |
| // multiple users but for each one will necessitate using VOP3, there will be | |
| // a code size increase. Try to avoid increasing code size unless we know it | |
| // will save on the instruction count. | |
| unsigned NumMayIncreaseSize = 0; | |
| Register Dst = MI.getOperand(0).getReg(); | |
| for (const MachineInstr &Use : MRI.use_nodbg_instructions(Dst)) { | |
| if (!hasSourceMods(Use)) | |
| return false; | |
| if (!opMustUseVOP3Encoding(Use, MRI)) { | |
| if (++NumMayIncreaseSize > CostThreshold) | |
| return false; | |
| } | |
| } | |
| return true; | |
| } | |
| static bool mayIgnoreSignedZero(const MachineInstr &MI) { | |
| return MI.getFlag(MachineInstr::MIFlag::FmNsz); | |
| } | |
| static bool isInv2Pi(const APFloat &APF) { | |
| static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118)); | |
| static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983)); | |
| static const APFloat KF64(APFloat::IEEEdouble(), | |
| APInt(64, 0x3fc45f306dc9c882)); | |
| return APF.bitwiseIsEqual(KF16) || APF.bitwiseIsEqual(KF32) || | |
| APF.bitwiseIsEqual(KF64); | |
| } | |
| // 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an | |
| // additional cost to negate them. | |
| static bool isConstantCostlierToNegate(MachineInstr &MI, Register Reg, | |
| MachineRegisterInfo &MRI) { | |
| std::optional<FPValueAndVReg> FPValReg; | |
| if (mi_match(Reg, MRI, m_GFCstOrSplat(FPValReg))) { | |
| if (FPValReg->Value.isZero() && !FPValReg->Value.isNegative()) | |
| return true; | |
| const GCNSubtarget &ST = MI.getMF()->getSubtarget<GCNSubtarget>(); | |
| if (ST.hasInv2PiInlineImm() && isInv2Pi(FPValReg->Value)) | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool AMDGPUCombinerHelper::canIgnoreLegacyMinMaxTies(const MachineInstr &MI, | |
| Register LHS, | |
| Register RHS) const { | |
| if (mayIgnoreSignedZero(MI)) | |
| return true; | |
| return VT && | |
| (VT->isKnownNeverLogicalZero(LHS) || VT->isKnownNeverLogicalZero(RHS)); | |
| } | |
| static unsigned inverseMinMax(unsigned Opc) { | |
| switch (Opc) { | |
| case AMDGPU::G_FMAXNUM: | |
| return AMDGPU::G_FMINNUM; | |
| case AMDGPU::G_FMINNUM: | |
| return AMDGPU::G_FMAXNUM; | |
| case AMDGPU::G_FMAXNUM_IEEE: | |
| return AMDGPU::G_FMINNUM_IEEE; | |
| case AMDGPU::G_FMINNUM_IEEE: | |
| return AMDGPU::G_FMAXNUM_IEEE; | |
| case AMDGPU::G_FMAXIMUM: | |
| return AMDGPU::G_FMINIMUM; | |
| case AMDGPU::G_FMINIMUM: | |
| return AMDGPU::G_FMAXIMUM; | |
| case AMDGPU::G_AMDGPU_FMAX_LEGACY: | |
| return AMDGPU::G_AMDGPU_FMIN_LEGACY; | |
| case AMDGPU::G_AMDGPU_FMIN_LEGACY: | |
| return AMDGPU::G_AMDGPU_FMAX_LEGACY; | |
| default: | |
| llvm_unreachable("invalid min/max opcode"); | |
| } | |
| } | |
| bool AMDGPUCombinerHelper::matchFoldableFneg(MachineInstr &MI, | |
| MachineInstr *&MatchInfo) const { | |
| Register Src = MI.getOperand(1).getReg(); | |
| MatchInfo = MRI.getVRegDef(Src); | |
| // If the input has multiple uses and we can either fold the negate down, or | |
| // the other uses cannot, give up. This both prevents unprofitable | |
| // transformations and infinite loops: we won't repeatedly try to fold around | |
| // a negate that has no 'good' form. | |
| if (MRI.hasOneNonDBGUse(Src)) { | |
| if (allUsesHaveSourceMods(MI, MRI, 0)) | |
| return false; | |
| } else { | |
| if (fnegFoldsIntoMI(*MatchInfo) && | |
| (allUsesHaveSourceMods(MI, MRI) || | |
| !allUsesHaveSourceMods(*MatchInfo, MRI))) | |
| return false; | |
| } | |
| switch (MatchInfo->getOpcode()) { | |
| case AMDGPU::G_AMDGPU_FMIN_LEGACY: | |
| case AMDGPU::G_AMDGPU_FMAX_LEGACY: | |
| if (isConstantCostlierToNegate(*MatchInfo, | |
| MatchInfo->getOperand(2).getReg(), MRI)) | |
| return false; | |
| // Swapping min<->max flips which operand a signed zero tie selects. | |
| return canIgnoreLegacyMinMaxTies(*MatchInfo, | |
| MatchInfo->getOperand(1).getReg(), | |
| MatchInfo->getOperand(2).getReg()); | |
| case AMDGPU::G_FMINNUM: | |
| case AMDGPU::G_FMAXNUM: | |
| case AMDGPU::G_FMINNUM_IEEE: | |
| case AMDGPU::G_FMAXNUM_IEEE: | |
| case AMDGPU::G_FMINIMUM: | |
| case AMDGPU::G_FMAXIMUM: | |
| // 0 doesn't have a negated inline immediate. | |
| return !isConstantCostlierToNegate(*MatchInfo, | |
| MatchInfo->getOperand(2).getReg(), MRI); | |
| case AMDGPU::G_FADD: | |
| case AMDGPU::G_FSUB: | |
| case AMDGPU::G_FMA: | |
| case AMDGPU::G_FMAD: | |
| return mayIgnoreSignedZero(*MatchInfo); | |
| case AMDGPU::G_FMUL: | |
| case AMDGPU::G_FPEXT: | |
| case AMDGPU::G_INTRINSIC_TRUNC: | |
| case AMDGPU::G_FPTRUNC: | |
| case AMDGPU::G_FRINT: | |
| case AMDGPU::G_FNEARBYINT: | |
| case AMDGPU::G_INTRINSIC_ROUND: | |
| case AMDGPU::G_INTRINSIC_ROUNDEVEN: | |
| case AMDGPU::G_FSIN: | |
| case AMDGPU::G_FCANONICALIZE: | |
| case AMDGPU::G_AMDGPU_RCP_IFLAG: | |
| return true; | |
| case AMDGPU::G_INTRINSIC: | |
| case AMDGPU::G_INTRINSIC_CONVERGENT: { | |
| Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MatchInfo)->getIntrinsicID(); | |
| switch (IntrinsicID) { | |
| case Intrinsic::amdgcn_rcp: | |
| case Intrinsic::amdgcn_rcp_legacy: | |
| case Intrinsic::amdgcn_sin: | |
| case Intrinsic::amdgcn_fmul_legacy: | |
| case Intrinsic::amdgcn_fmed3: | |
| return true; | |
| case Intrinsic::amdgcn_fma_legacy: | |
| return mayIgnoreSignedZero(*MatchInfo); | |
| default: | |
| return false; | |
| } | |
| } | |
| default: | |
| return false; | |
| } | |
| } | |
| void AMDGPUCombinerHelper::applyFoldableFneg(MachineInstr &MI, | |
| MachineInstr *&MatchInfo) const { | |
| // Transform: | |
| // %A = inst %Op1, ... | |
| // %B = fneg %A | |
| // | |
| // into: | |
| // | |
| // (if %A has one use, specifically fneg above) | |
| // %B = inst (maybe fneg %Op1), ... | |
| // | |
| // (if %A has multiple uses) | |
| // %B = inst (maybe fneg %Op1), ... | |
| // %A = fneg %B | |
| // Replace register in operand with a register holding negated value. | |
| auto NegateOperand = [&](MachineOperand &Op) { | |
| Register Reg = Op.getReg(); | |
| if (!mi_match(Reg, MRI, m_GFNeg(m_Reg(Reg)))) | |
| Reg = Builder.buildFNeg(MRI.getType(Reg), Reg).getReg(0); | |
| replaceRegOpWith(MRI, Op, Reg); | |
| }; | |
| // Replace either register in operands with a register holding negated value. | |
| auto NegateEitherOperand = [&](MachineOperand &X, MachineOperand &Y) { | |
| Register XReg = X.getReg(); | |
| Register YReg = Y.getReg(); | |
| if (mi_match(XReg, MRI, m_GFNeg(m_Reg(XReg)))) | |
| replaceRegOpWith(MRI, X, XReg); | |
| else if (mi_match(YReg, MRI, m_GFNeg(m_Reg(YReg)))) | |
| replaceRegOpWith(MRI, Y, YReg); | |
| else { | |
| YReg = Builder.buildFNeg(MRI.getType(YReg), YReg).getReg(0); | |
| replaceRegOpWith(MRI, Y, YReg); | |
| } | |
| }; | |
| Builder.setInstrAndDebugLoc(*MatchInfo); | |
| // Negate appropriate operands so that resulting value of MatchInfo is | |
| // negated. | |
| switch (MatchInfo->getOpcode()) { | |
| case AMDGPU::G_FADD: | |
| case AMDGPU::G_FSUB: | |
| NegateOperand(MatchInfo->getOperand(1)); | |
| NegateOperand(MatchInfo->getOperand(2)); | |
| break; | |
| case AMDGPU::G_FMUL: | |
| NegateEitherOperand(MatchInfo->getOperand(1), MatchInfo->getOperand(2)); | |
| break; | |
| case AMDGPU::G_FMINNUM: | |
| case AMDGPU::G_FMAXNUM: | |
| case AMDGPU::G_FMINNUM_IEEE: | |
| case AMDGPU::G_FMAXNUM_IEEE: | |
| case AMDGPU::G_FMINIMUM: | |
| case AMDGPU::G_FMAXIMUM: | |
| case AMDGPU::G_AMDGPU_FMIN_LEGACY: | |
| case AMDGPU::G_AMDGPU_FMAX_LEGACY: { | |
| NegateOperand(MatchInfo->getOperand(1)); | |
| NegateOperand(MatchInfo->getOperand(2)); | |
| unsigned Opposite = inverseMinMax(MatchInfo->getOpcode()); | |
| replaceOpcodeWith(*MatchInfo, Opposite); | |
| break; | |
| } | |
| case AMDGPU::G_FMA: | |
| case AMDGPU::G_FMAD: | |
| NegateEitherOperand(MatchInfo->getOperand(1), MatchInfo->getOperand(2)); | |
| NegateOperand(MatchInfo->getOperand(3)); | |
| break; | |
| case AMDGPU::G_FPEXT: | |
| case AMDGPU::G_INTRINSIC_TRUNC: | |
| case AMDGPU::G_FRINT: | |
| case AMDGPU::G_FNEARBYINT: | |
| case AMDGPU::G_INTRINSIC_ROUND: | |
| case AMDGPU::G_INTRINSIC_ROUNDEVEN: | |
| case AMDGPU::G_FSIN: | |
| case AMDGPU::G_FCANONICALIZE: | |
| case AMDGPU::G_AMDGPU_RCP_IFLAG: | |
| case AMDGPU::G_FPTRUNC: | |
| NegateOperand(MatchInfo->getOperand(1)); | |
| break; | |
| case AMDGPU::G_INTRINSIC: | |
| case AMDGPU::G_INTRINSIC_CONVERGENT: { | |
| Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MatchInfo)->getIntrinsicID(); | |
| switch (IntrinsicID) { | |
| case Intrinsic::amdgcn_rcp: | |
| case Intrinsic::amdgcn_rcp_legacy: | |
| case Intrinsic::amdgcn_sin: | |
| NegateOperand(MatchInfo->getOperand(2)); | |
| break; | |
| case Intrinsic::amdgcn_fmul_legacy: | |
| NegateEitherOperand(MatchInfo->getOperand(2), MatchInfo->getOperand(3)); | |
| break; | |
| case Intrinsic::amdgcn_fmed3: | |
| NegateOperand(MatchInfo->getOperand(2)); | |
| NegateOperand(MatchInfo->getOperand(3)); | |
| NegateOperand(MatchInfo->getOperand(4)); | |
| break; | |
| case Intrinsic::amdgcn_fma_legacy: | |
| NegateEitherOperand(MatchInfo->getOperand(2), MatchInfo->getOperand(3)); | |
| NegateOperand(MatchInfo->getOperand(4)); | |
| break; | |
| default: | |
| llvm_unreachable("folding fneg not supported for this intrinsic"); | |
| } | |
| break; | |
| } | |
| default: | |
| llvm_unreachable("folding fneg not supported for this instruction"); | |
| } | |
| Register Dst = MI.getOperand(0).getReg(); | |
| Register MatchInfoDst = MatchInfo->getOperand(0).getReg(); | |
| if (MRI.hasOneNonDBGUse(MatchInfoDst)) { | |
| // MatchInfo now has negated value so use that instead of old Dst. | |
| replaceRegWith(MRI, Dst, MatchInfoDst); | |
| } else { | |
| // We want to swap all uses of Dst with uses of MatchInfoDst and vice versa | |
| // but replaceRegWith will replace defs as well. It is easier to replace one | |
| // def with a new register. | |
| LLT Type = MRI.getType(Dst); | |
| Register NegatedMatchInfo = MRI.createGenericVirtualRegister(Type); | |
| replaceRegOpWith(MRI, MatchInfo->getOperand(0), NegatedMatchInfo); | |
| // MatchInfo now has negated value so use that instead of old Dst. | |
| replaceRegWith(MRI, Dst, NegatedMatchInfo); | |
| // Recreate non negated value for other uses of old MatchInfoDst | |
| auto NextInst = ++MatchInfo->getIterator(); | |
| Builder.setInstrAndDebugLoc(*NextInst); | |
| Builder.buildFNeg(MatchInfoDst, NegatedMatchInfo, MI.getFlags()); | |
| } | |
| MI.eraseFromParent(); | |
| } | |
| bool AMDGPUCombinerHelper::matchFoldFAbsFptrunc(MachineInstr &Fabs, | |
| MachineInstr &Fptrunc) const { | |
| Register Round = Fptrunc.getOperand(0).getReg(); | |
| if (!MRI.hasOneNonDBGUse(Round)) | |
| return false; | |
| LLT SrcTy = MRI.getType(Fptrunc.getOperand(1).getReg()); | |
| return isLegalOrBeforeLegalizer({TargetOpcode::G_FABS, {SrcTy}}); | |
| } | |
| void AMDGPUCombinerHelper::applyFoldFAbsFptrunc(MachineInstr &Fabs, | |
| MachineInstr &Fptrunc) const { | |
| // fabs (fptrunc x) -> fptrunc (fabs x) | |
| Register Dst = Fabs.getOperand(0).getReg(); | |
| Register Src = Fptrunc.getOperand(1).getReg(); | |
| Builder.setInstrAndDebugLoc(Fabs); | |
| Register Abs = | |
| Builder.buildFAbs(MRI.getType(Src), Src, Fabs.getFlags()).getReg(0); | |
| Builder.buildFPTrunc(Dst, Abs, Fptrunc.getFlags()); | |
| Fabs.eraseFromParent(); | |
| } | |
| // TODO: Should return converted value / extension source and avoid introducing | |
| // intermediate fptruncs in the apply function. | |
| static bool isFPExtFromF16OrConst(const MachineRegisterInfo &MRI, | |
| Register Reg) { | |
| Register SrcReg; | |
| if (mi_match(Reg, MRI, m_GFPExt(m_Reg(SrcReg)))) | |
| return MRI.getType(SrcReg) == LLT::float16(); | |
| const ConstantFP *FPImm; | |
| if (mi_match(Reg, MRI, m_GFCst(FPImm))) { | |
| APFloat Val = FPImm->getValueAPF(); | |
| bool LosesInfo = true; | |
| Val.convert(APFloat::IEEEhalf(), APFloat::rmNearestTiesToEven, &LosesInfo); | |
| return !LosesInfo; | |
| } | |
| return false; | |
| } | |
| bool AMDGPUCombinerHelper::matchExpandPromotedF16FMed3(MachineInstr &MI, | |
| Register Src0, | |
| Register Src1, | |
| Register Src2) const { | |
| assert(MI.getOpcode() == TargetOpcode::G_FPTRUNC); | |
| Register SrcReg = MI.getOperand(1).getReg(); | |
| if (MRI.getType(MI.getOperand(0).getReg()) != LLT::float16()) | |
| return false; | |
| if (!MRI.hasOneNonDBGUse(SrcReg) || MRI.getType(SrcReg) != LLT::float32()) | |
| return false; | |
| return isFPExtFromF16OrConst(MRI, Src0) && isFPExtFromF16OrConst(MRI, Src1) && | |
| isFPExtFromF16OrConst(MRI, Src2); | |
| } | |
| void AMDGPUCombinerHelper::applyExpandPromotedF16FMed3(MachineInstr &MI, | |
| Register Src0, | |
| Register Src1, | |
| Register Src2) const { | |
| // We expect fptrunc (fpext x) to fold out, and to constant fold any constant | |
| // sources. | |
| Src0 = Builder.buildFPTrunc(LLT::float16(), Src0).getReg(0); | |
| Src1 = Builder.buildFPTrunc(LLT::float16(), Src1).getReg(0); | |
| Src2 = Builder.buildFPTrunc(LLT::float16(), Src2).getReg(0); | |
| LLT Ty = MRI.getType(Src0); | |
| auto A1 = Builder.buildFMinNumIEEE(Ty, Src0, Src1); | |
| auto B1 = Builder.buildFMaxNumIEEE(Ty, Src0, Src1); | |
| auto C1 = Builder.buildFMaxNumIEEE(Ty, A1, Src2); | |
| Builder.buildFMinNumIEEE(MI.getOperand(0), B1, C1); | |
| MI.eraseFromParent(); | |
| } | |
| bool AMDGPUCombinerHelper::matchCombineFmulWithSelectToFldexp( | |
| MachineInstr &MI, MachineInstr &Sel, | |
| std::function<void(MachineIRBuilder &)> &MatchInfo) const { | |
| assert(MI.getOpcode() == TargetOpcode::G_FMUL); | |
| assert(Sel.getOpcode() == TargetOpcode::G_SELECT); | |
| assert(MI.getOperand(2).getReg() == Sel.getOperand(0).getReg()); | |
| Register Dst = MI.getOperand(0).getReg(); | |
| LLT DestTy = MRI.getType(Dst); | |
| LLT ScalarDestTy = DestTy.getScalarType(); | |
| // TODO: Expected float type in ScalarDestTy | |
| if ((ScalarDestTy != LLT::float64() && ScalarDestTy != LLT::float32() && | |
| ScalarDestTy != LLT::float16()) || | |
| !MRI.hasOneNonDBGUse(Sel.getOperand(0).getReg())) | |
| return false; | |
| Register SelectCondReg = Sel.getOperand(1).getReg(); | |
| Register SelectTrueReg = Sel.getOperand(2).getReg(); | |
| Register SelectFalseReg = Sel.getOperand(3).getReg(); | |
| const auto SelectTrueVal = | |
| isConstantOrConstantSplatVectorFP(SelectTrueReg, MRI); | |
| if (!SelectTrueVal) | |
| return false; | |
| const auto SelectFalseVal = | |
| isConstantOrConstantSplatVectorFP(SelectFalseReg, MRI); | |
| if (!SelectFalseVal) | |
| return false; | |
| if (SelectTrueVal->isNegative() != SelectFalseVal->isNegative()) | |
| return false; | |
| // For f32, only non-inline constants should be transformed. | |
| // TODO: Expected float32 | |
| if (ScalarDestTy == LLT::float32() && TII.isInlineConstant(*SelectTrueVal) && | |
| TII.isInlineConstant(*SelectFalseVal)) | |
| return false; | |
| int SelectTrueLog2Val = SelectTrueVal->getExactLog2Abs(); | |
| if (SelectTrueLog2Val == INT_MIN) | |
| return false; | |
| int SelectFalseLog2Val = SelectFalseVal->getExactLog2Abs(); | |
| if (SelectFalseLog2Val == INT_MIN) | |
| return false; | |
| MatchInfo = [=, &MI](MachineIRBuilder &Builder) { | |
| LLT IntDestTy = DestTy.changeElementType(LLT::integer(32)); | |
| auto NewSel = Builder.buildSelect( | |
| IntDestTy, SelectCondReg, | |
| Builder.buildConstant(IntDestTy, SelectTrueLog2Val), | |
| Builder.buildConstant(IntDestTy, SelectFalseLog2Val)); | |
| Register XReg = MI.getOperand(1).getReg(); | |
| if (SelectTrueVal->isNegative()) { | |
| auto NegX = | |
| Builder.buildFNeg(DestTy, XReg, MRI.getVRegDef(XReg)->getFlags()); | |
| Builder.buildFLdexp(Dst, NegX, NewSel, MI.getFlags()); | |
| } else { | |
| Builder.buildFLdexp(Dst, XReg, NewSel, MI.getFlags()); | |
| } | |
| }; | |
| return true; | |
| } | |
| bool AMDGPUCombinerHelper::matchConstantIs32BitMask(Register Reg) const { | |
| auto Res = getIConstantVRegValWithLookThrough(Reg, MRI); | |
| if (!Res) | |
| return false; | |
| const uint64_t Val = Res->Value.getZExtValue(); | |
| unsigned MaskIdx = 0; | |
| unsigned MaskLen = 0; | |
| if (!isShiftedMask_64(Val, MaskIdx, MaskLen)) | |
| return false; | |
| // Check if low 32 bits or high 32 bits are all ones. | |
| return MaskLen >= 32 && ((MaskIdx == 0) || (MaskIdx == 64 - MaskLen)); | |
| } | |
| bool AMDGPUCombinerHelper::matchBinopWith32BitMask(Register X, | |
| Register Y) const { | |
| // If both operands are constants, constant_fold_binop can fold the whole | |
| // binop to a single constant, which is strictly better than narrowing it | |
| // into two s32 ops. Bail so the fold is not preempted. | |
| if (getIConstantVRegValWithLookThrough(X, MRI) && | |
| getIConstantVRegValWithLookThrough(Y, MRI)) | |
| return false; | |
| return matchConstantIs32BitMask(X) || matchConstantIs32BitMask(Y); | |
| } | |
| bool AMDGPUCombinerHelper::isCarryLikeBool(Register Reg, unsigned Depth) const { | |
| constexpr unsigned MaxDepth = 6; | |
| const MachineInstr *Def = MRI.getVRegDef(Reg); | |
| if (!Def || Depth > MaxDepth) | |
| return false; | |
| switch (Def->getOpcode()) { | |
| case TargetOpcode::G_ICMP: | |
| case TargetOpcode::G_FCMP: | |
| case TargetOpcode::G_IS_FPCLASS: | |
| return true; | |
| case TargetOpcode::G_INTRINSIC: | |
| return cast<GIntrinsic>(Def)->is(Intrinsic::amdgcn_class); | |
| case TargetOpcode::G_UADDO: | |
| case TargetOpcode::G_UADDE: | |
| case TargetOpcode::G_USUBO: | |
| case TargetOpcode::G_USUBE: | |
| return Def->getOperand(1).getReg() == Reg; | |
| case TargetOpcode::G_XOR: | |
| // A not of a carry-like boolean. | |
| if (mi_match(Def->getOperand(2).getReg(), MRI, m_AllOnesInt())) | |
| return isCarryLikeBool(Def->getOperand(1).getReg(), Depth + 1); | |
| [[fallthrough]]; | |
| case TargetOpcode::G_AND: | |
| case TargetOpcode::G_OR: | |
| return isCarryLikeBool(Def->getOperand(1).getReg(), Depth + 1) && | |
| isCarryLikeBool(Def->getOperand(2).getReg(), Depth + 1); | |
| default: | |
| return false; | |
| } | |
| } | |
| static bool isBoolType(LLT Ty) { return Ty.isScalar(1); } | |
| bool AMDGPUCombinerHelper::isFoldableCarryExt(Register Reg, Register *Carry, | |
| bool *IsSExt) const { | |
| Register Src; | |
| bool SExt; | |
| // The high bits of an anyext are undefined, so it can be treated as a zext. | |
| if (mi_match(Reg, MRI, m_any_of(m_GZExt(m_Reg(Src)), m_GAnyExt(m_Reg(Src))))) | |
| SExt = false; | |
| else if (mi_match(Reg, MRI, m_GSExt(m_Reg(Src)))) | |
| SExt = true; | |
| else | |
| return false; | |
| // Any other boolean must first be copied to SCC or VCC, which costs about | |
| // as much as the extension. | |
| if (!isBoolType(MRI.getType(Src)) || !MRI.hasOneNonDBGUse(Reg) || | |
| !isCarryLikeBool(Src)) | |
| return false; | |
| if (Carry) | |
| *Carry = Src; | |
| if (IsSExt) | |
| *IsSExt = SExt; | |
| return true; | |
| } | |
| bool AMDGPUCombinerHelper::matchConditionalCarryOp( | |
| MachineInstr &MI, ConditionalCarryOp &MatchInfo) const { | |
| LLT Ty = MRI.getType(MI.getOperand(0).getReg()); | |
| if (!Ty.isScalar() || Ty.isFloat()) | |
| return false; | |
| unsigned Size = Ty.getSizeInBits(); | |
| // Most 16/32-bit cases come from div/rem legalization, and the legalizer | |
| // splits a 64-bit add on most subtargets, hiding the x +/- 1 operand. | |
| if (isPreLegalize() ? Size != 64 : (Size != 16 && Size != 32)) | |
| return false; | |
| // A uniform 64-bit add or select does not clobber SCC on a subtarget with | |
| // 64-bit scalar adds, but the two carry ops that replace it do, so a | |
| // condition with other users would have to be saved and restored. | |
| // Uniformity is not known before legalization, so a condition with other | |
| // users is rejected for all 64-bit types on such a subtarget. | |
| bool NeedsSingleUseCarry = Size == 64 && STI.hasScalarAddSub64(); | |
| // The base is folded into the add/sub-with-carry, so a known zero base is | |
| // better left to the identity combines. | |
| auto IsZero = [&](Register Reg) { | |
| return mi_match(Reg, MRI, m_SpecificICst(0)); | |
| }; | |
| // Ext is a zext or sext of a boolean that is used as an addend of an add | |
| // (IsAddRoot) or as the subtrahend of a sub. | |
| auto MatchExt = [&](Register Ext, Register Base, bool IsAddRoot) { | |
| Register Carry; | |
| bool IsSExt; | |
| if (IsZero(Base) || !isFoldableCarryExt(Ext, &Carry, &IsSExt) || | |
| (NeedsSingleUseCarry && !MRI.hasOneNonDBGUse(Carry))) | |
| return false; | |
| MatchInfo = {IsAddRoot != IsSExt, /*K=*/0, Base, Carry}; | |
| return true; | |
| }; | |
| switch (MI.getOpcode()) { | |
| case TargetOpcode::G_ADD: { | |
| Register LHS = MI.getOperand(1).getReg(); | |
| Register RHS = MI.getOperand(2).getReg(); | |
| // When both operands are carries, fold only the right one, so that the | |
| // two carries do not have to be live at the same time. | |
| return MatchExt(RHS, LHS, /*IsAddRoot=*/true) || | |
| MatchExt(LHS, RHS, /*IsAddRoot=*/true); | |
| } | |
| case TargetOpcode::G_SUB: | |
| return MatchExt(MI.getOperand(2).getReg(), MI.getOperand(1).getReg(), | |
| /*IsAddRoot=*/false); | |
| case TargetOpcode::G_SELECT: { | |
| Register Cond = MI.getOperand(1).getReg(); | |
| Register TrueReg = MI.getOperand(2).getReg(); | |
| Register FalseReg = MI.getOperand(3).getReg(); | |
| if (!isBoolType(MRI.getType(Cond)) || | |
| (NeedsSingleUseCarry && !MRI.hasOneNonDBGUse(Cond))) | |
| return false; | |
| // The select already needs its condition in SCC or VCC, which is the | |
| // form that the carry-in operand needs, so any condition is accepted. | |
| // Unlike in SelectionDAG, a uniform condition with other users is | |
| // already saved in an SGPR and copied back to SCC for each user, so a | |
| // 16-bit or 32-bit carry op does not add a save and restore. | |
| auto MatchStep = [&](Register Step, Register Base, bool StepIfTrue) { | |
| if (!MRI.hasOneNonDBGUse(Step)) | |
| return false; | |
| int64_t Imm; | |
| Register Narrow; | |
| if (mi_match(Step, MRI, m_GAnyExt(m_Reg(Narrow)))) { | |
| // A narrow select that was widened by the legalizer: | |
| // (select c, (anyext (add (trunc x), Imm)), x) | |
| // The high bits of the anyext are undefined, so x + Imm can replace | |
| // the step. | |
| if (!MRI.hasOneNonDBGUse(Narrow) || | |
| !mi_match(Narrow, MRI, | |
| m_GAdd(m_GTrunc(m_SpecificReg(Base)), m_ICst(Imm)))) | |
| return false; | |
| } else if (!mi_match(Step, MRI, | |
| m_GAdd(m_SpecificReg(Base), m_ICst(Imm)))) { | |
| return false; | |
| } | |
| if (Imm != 1 && Imm != -1) | |
| return false; | |
| // (select c, x + Imm, x) = x + Imm * c | |
| // (select c, x, x + Imm) = x + Imm - Imm * c | |
| if (StepIfTrue) | |
| MatchInfo = {Imm == 1, /*K=*/0, Base, Cond}; | |
| else | |
| MatchInfo = {Imm == -1, /*K=*/-1, Base, Cond}; | |
| return true; | |
| }; | |
| return MatchStep(TrueReg, FalseReg, /*StepIfTrue=*/true) || | |
| MatchStep(FalseReg, TrueReg, /*StepIfTrue=*/false); | |
| } | |
| default: | |
| return false; | |
| } | |
| } | |
| void AMDGPUCombinerHelper::applyConditionalCarryOp( | |
| MachineInstr &MI, const ConditionalCarryOp &MatchInfo) const { | |
| assert((MatchInfo.K == 0 || MatchInfo.K == -1) && | |
| "unexpected conditional carry op constant"); | |
| Builder.setInstrAndDebugLoc(MI); | |
| Register Dst = MI.getOperand(0).getReg(); | |
| LLT Ty = MRI.getType(Dst); | |
| LLT Ty32 = Ty.changeElementSize(32); | |
| LLT CarryTy = MRI.getType(MatchInfo.Carry); | |
| unsigned Opc = | |
| MatchInfo.IsAdd ? TargetOpcode::G_UADDE : TargetOpcode::G_USUBE; | |
| Register K = Builder.buildConstant(Ty32, MatchInfo.K).getReg(0); | |
| switch (Ty.getSizeInBits()) { | |
| case 16: { | |
| // Only the low 16 bits of the 32-bit result are used. | |
| auto Base = Builder.buildAnyExt(Ty32, MatchInfo.Base); | |
| auto Op = | |
| Builder.buildInstr(Opc, {Ty32, CarryTy}, {Base, K, MatchInfo.Carry}); | |
| Builder.buildTrunc(Dst, Op.getReg(0)); | |
| break; | |
| } | |
| case 32: | |
| Builder.buildInstr(Opc, {Dst, CarryTy}, | |
| {MatchInfo.Base, K, MatchInfo.Carry}); | |
| break; | |
| case 64: { | |
| // K is 0 or -1, so both halves of the 64-bit constant are K. | |
| auto Base = Builder.buildUnmerge(Ty32, MatchInfo.Base); | |
| auto Lo = Builder.buildInstr(Opc, {Ty32, CarryTy}, | |
| {Base.getReg(0), K, MatchInfo.Carry}); | |
| auto Hi = Builder.buildInstr(Opc, {Ty32, CarryTy}, | |
| {Base.getReg(1), K, Lo.getReg(1)}); | |
| Builder.buildMergeLikeInstr(Dst, {Lo.getReg(0), Hi.getReg(0)}); | |
| break; | |
| } | |
| default: | |
| llvm_unreachable("unexpected conditional carry op type"); | |
| } | |
| MI.eraseFromParent(); | |
| } |