[LV] Remove unused getReductionPatternCost (NFC) (#221183)
In-loop reduction handling + costing has been completely migrated to
VPlan. Remove unused legacy getReductionPatternCost.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index dc42759..951e67b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1276,12 +1276,6 @@
/// width. Vector width of one means scalar.
InstructionCost getInstructionCost(Instruction *I, ElementCount VF);
- /// Return the cost of instructions in an inloop reduction pattern, if I is
- /// part of that pattern.
- std::optional<InstructionCost> getReductionPatternCost(Instruction *I,
- ElementCount VF,
- Type *VectorTy) const;
-
/// Returns true if \p Op should be considered invariant and if it is
/// trivially hoistable.
bool shouldConsiderInvariant(Value *Op);
@@ -4331,193 +4325,6 @@
return Cost;
}
-std::optional<InstructionCost>
-LoopVectorizationCostModel::getReductionPatternCost(Instruction *I,
- ElementCount VF,
- Type *Ty) const {
- using namespace llvm::PatternMatch;
- // Early exit for no inloop reductions
- if (Config.getInLoopReductions().empty() || VF.isScalar() ||
- !isa<VectorType>(Ty))
- return std::nullopt;
- auto *VectorTy = cast<VectorType>(Ty);
-
- // We are looking for a pattern of, and finding the minimal acceptable cost:
- // reduce(mul(ext(A), ext(B))) or
- // reduce(mul(A, B)) or
- // reduce(ext(A)) or
- // reduce(A).
- // The basic idea is that we walk down the tree to do that, finding the root
- // reduction instruction in InLoopReductionImmediateChains. From there we find
- // the pattern of mul/ext and test the cost of the entire pattern vs the cost
- // of the components. If the reduction cost is lower then we return it for the
- // reduction instruction and 0 for the other instructions in the pattern. If
- // it is not we return an invalid cost specifying the orignal cost method
- // should be used.
- Instruction *RetI = I;
- if (match(RetI, m_ZExtOrSExt(m_Value()))) {
- if (!RetI->hasOneUser())
- return std::nullopt;
- RetI = RetI->user_back();
- }
-
- if (match(RetI, m_OneUse(m_Mul(m_Value(), m_Value()))) &&
- RetI->user_back()->getOpcode() == Instruction::Add) {
- RetI = RetI->user_back();
- }
-
- // Test if the found instruction is a reduction, and if not return an invalid
- // cost specifying the parent to use the original cost modelling.
- Instruction *LastChain = Config.getInLoopReductionImmediateChain(RetI);
- if (!LastChain)
- return std::nullopt;
-
- // Find the reduction this chain is a part of and calculate the basic cost of
- // the reduction on its own.
- Instruction *ReductionPhi = LastChain;
- while (!isa<PHINode>(ReductionPhi))
- ReductionPhi = Config.getInLoopReductionImmediateChain(ReductionPhi);
-
- const RecurrenceDescriptor &RdxDesc =
- Legal->getRecurrenceDescriptor(cast<PHINode>(ReductionPhi));
-
- InstructionCost BaseCost;
- RecurKind RK = RdxDesc.getRecurrenceKind();
- if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
- Intrinsic::ID MinMaxID = getMinMaxReductionIntrinsicOp(RK);
- BaseCost = TTI.getMinMaxReductionCost(
- MinMaxID, VectorTy, RdxDesc.getFastMathFlags(), Config.CostKind);
- } else {
- BaseCost = TTI.getArithmeticReductionCost(RdxDesc.getOpcode(), VectorTy,
- RdxDesc.getFastMathFlags(),
- Config.CostKind);
- }
-
- // For a call to the llvm.fmuladd intrinsic we need to add the cost of a
- // normal fmul instruction to the cost of the fadd reduction.
- if (RK == RecurKind::FMulAdd)
- BaseCost += TTI.getArithmeticInstrCost(Instruction::FMul, VectorTy,
- Config.CostKind);
-
- // If we're using ordered reductions then we can just return the base cost
- // here, since getArithmeticReductionCost calculates the full ordered
- // reduction cost when FP reassociation is not allowed.
- if (Config.useOrderedReductions(RdxDesc))
- return BaseCost;
-
- // Get the operand that was not the reduction chain and match it to one of the
- // patterns, returning the better cost if it is found.
- Instruction *RedOp = RetI->getOperand(1) == LastChain
- ? dyn_cast<Instruction>(RetI->getOperand(0))
- : dyn_cast<Instruction>(RetI->getOperand(1));
-
- VectorTy = VectorType::get(I->getOperand(0)->getType(), VectorTy);
-
- Instruction *Op0, *Op1;
- if (RedOp && RdxDesc.getOpcode() == Instruction::Add &&
- match(RedOp,
- m_ZExtOrSExt(m_Mul(m_Instruction(Op0), m_Instruction(Op1)))) &&
- match(Op0, m_ZExtOrSExt(m_Value())) &&
- Op0->getOpcode() == Op1->getOpcode() &&
- Op0->getOperand(0)->getType() == Op1->getOperand(0)->getType() &&
- !TheLoop->isLoopInvariant(Op0) && !TheLoop->isLoopInvariant(Op1) &&
- (Op0->getOpcode() == RedOp->getOpcode() || Op0 == Op1)) {
-
- // Matched reduce.add(ext(mul(ext(A), ext(B)))
- // Note that the extend opcodes need to all match, or if A==B they will have
- // been converted to zext(mul(sext(A), sext(A))) as it is known positive,
- // which is equally fine.
- bool IsUnsigned = isa<ZExtInst>(Op0);
- auto *ExtType = VectorType::get(Op0->getOperand(0)->getType(), VectorTy);
- auto *MulType = VectorType::get(Op0->getType(), VectorTy);
-
- InstructionCost ExtCost =
- TTI.getCastInstrCost(Op0->getOpcode(), MulType, ExtType,
- TTI::CastContextHint::None, Config.CostKind, Op0);
- InstructionCost MulCost =
- TTI.getArithmeticInstrCost(Instruction::Mul, MulType, Config.CostKind);
- InstructionCost Ext2Cost = TTI.getCastInstrCost(
- RedOp->getOpcode(), VectorTy, MulType, TTI::CastContextHint::None,
- Config.CostKind, RedOp);
-
- InstructionCost RedCost = TTI.getMulAccReductionCost(
- IsUnsigned, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), ExtType,
- Config.CostKind);
-
- if (RedCost.isValid() &&
- RedCost < ExtCost * 2 + MulCost + Ext2Cost + BaseCost)
- return I == RetI ? RedCost : 0;
- } else if (RedOp && match(RedOp, m_ZExtOrSExt(m_Value())) &&
- !TheLoop->isLoopInvariant(RedOp)) {
- // Matched reduce(ext(A))
- bool IsUnsigned = isa<ZExtInst>(RedOp);
- auto *ExtType = VectorType::get(RedOp->getOperand(0)->getType(), VectorTy);
- InstructionCost RedCost = TTI.getExtendedReductionCost(
- RdxDesc.getOpcode(), IsUnsigned, RdxDesc.getRecurrenceType(), ExtType,
- RdxDesc.getFastMathFlags(), Config.CostKind);
-
- InstructionCost ExtCost = TTI.getCastInstrCost(
- RedOp->getOpcode(), VectorTy, ExtType, TTI::CastContextHint::None,
- Config.CostKind, RedOp);
- if (RedCost.isValid() && RedCost < BaseCost + ExtCost)
- return I == RetI ? RedCost : 0;
- } else if (RedOp && RdxDesc.getOpcode() == Instruction::Add &&
- match(RedOp, m_Mul(m_Instruction(Op0), m_Instruction(Op1)))) {
- if (match(Op0, m_ZExtOrSExt(m_Value())) &&
- Op0->getOpcode() == Op1->getOpcode() &&
- !TheLoop->isLoopInvariant(Op0) && !TheLoop->isLoopInvariant(Op1)) {
- bool IsUnsigned = isa<ZExtInst>(Op0);
- Type *Op0Ty = Op0->getOperand(0)->getType();
- Type *Op1Ty = Op1->getOperand(0)->getType();
- Type *LargestOpTy =
- Op0Ty->getIntegerBitWidth() < Op1Ty->getIntegerBitWidth() ? Op1Ty
- : Op0Ty;
- auto *ExtType = VectorType::get(LargestOpTy, VectorTy);
-
- // Matched reduce.add(mul(ext(A), ext(B))), where the two ext may be of
- // different sizes. We take the largest type as the ext to reduce, and add
- // the remaining cost as, for example reduce(mul(ext(ext(A)), ext(B))).
- InstructionCost ExtCost0 = TTI.getCastInstrCost(
- Op0->getOpcode(), VectorTy, VectorType::get(Op0Ty, VectorTy),
- TTI::CastContextHint::None, Config.CostKind, Op0);
- InstructionCost ExtCost1 = TTI.getCastInstrCost(
- Op1->getOpcode(), VectorTy, VectorType::get(Op1Ty, VectorTy),
- TTI::CastContextHint::None, Config.CostKind, Op1);
- InstructionCost MulCost = TTI.getArithmeticInstrCost(
- Instruction::Mul, VectorTy, Config.CostKind);
-
- InstructionCost RedCost = TTI.getMulAccReductionCost(
- IsUnsigned, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), ExtType,
- Config.CostKind);
- InstructionCost ExtraExtCost = 0;
- if (Op0Ty != LargestOpTy || Op1Ty != LargestOpTy) {
- Instruction *ExtraExtOp = (Op0Ty != LargestOpTy) ? Op0 : Op1;
- ExtraExtCost = TTI.getCastInstrCost(
- ExtraExtOp->getOpcode(), ExtType,
- VectorType::get(ExtraExtOp->getOperand(0)->getType(), VectorTy),
- TTI::CastContextHint::None, Config.CostKind, ExtraExtOp);
- }
-
- if (RedCost.isValid() &&
- (RedCost + ExtraExtCost) < (ExtCost0 + ExtCost1 + MulCost + BaseCost))
- return I == RetI ? RedCost : 0;
- } else if (!match(I, m_ZExtOrSExt(m_Value()))) {
- // Matched reduce.add(mul())
- InstructionCost MulCost = TTI.getArithmeticInstrCost(
- Instruction::Mul, VectorTy, Config.CostKind);
-
- InstructionCost RedCost = TTI.getMulAccReductionCost(
- true, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), VectorTy,
- Config.CostKind);
-
- if (RedCost.isValid() && RedCost < MulCost + BaseCost)
- return I == RetI ? RedCost : 0;
- }
- }
-
- return I == RetI ? std::optional<InstructionCost>(BaseCost) : std::nullopt;
-}
-
InstructionCost
LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
ElementCount VF) {
@@ -5061,10 +4868,6 @@
PSE.getSCEV(I->getOperand(1))->isOne())))
return 0;
- // Detect reduction patterns
- if (auto RedCost = getReductionPatternCost(I, VF, VectorTy))
- return *RedCost;
-
// Certain instructions can be cheaper to vectorize if they have a constant
// second vector operand. One example of this are shifts on x86.
Value *Op2 = I->getOperand(1);
@@ -5227,10 +5030,6 @@
Trunc);
}
- // Detect reduction patterns
- if (auto RedCost = getReductionPatternCost(I, VF, VectorTy))
- return *RedCost;
-
Type *SrcScalarTy = I->getOperand(0)->getType();
Instruction *Op0AsInstruction = dyn_cast<Instruction>(I->getOperand(0));
if (canTruncateToMinimalBitwidth(Op0AsInstruction, VF))
