[VPlan] Track IsOrdered in VPReductionRecipe, remove use of ILV (NFCI).
Instead of using ILV.useOrderedReductions during ::execute, instead
store the information at recipe construction.
Another step towards making recipe'::execute independent of legacy ILV.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 49bacb5..9e22dce 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -573,10 +573,6 @@
/// Fix the non-induction PHIs in \p Plan.
void fixNonInductionPHIs(VPlan &Plan, VPTransformState &State);
- /// Returns true if the reordering of FP operations is not allowed, but we are
- /// able to vectorize with strict in-order reductions for the given RdxDesc.
- bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc);
-
/// Create a new phi node for the induction variable \p OrigPhi to resume
/// iteration count in the scalar epilogue, from where the vectorized loop
/// left off. \p Step is the SCEV-expanded induction step to use. In cases
@@ -3714,11 +3710,6 @@
}
}
-bool InnerLoopVectorizer::useOrderedReductions(
- const RecurrenceDescriptor &RdxDesc) {
- return Cost->useOrderedReductions(RdxDesc);
-}
-
void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
// We should not collect Scalars more than once per VF. Right now, this
// function is called from collectUniformsAndScalars(), which already does
@@ -9056,8 +9047,9 @@
if (CM.blockNeedsPredicationForAnyReason(BB))
CondOp = RecipeBuilder.getBlockInMask(BB);
- VPReductionRecipe *RedRecipe = new VPReductionRecipe(
- RdxDesc, CurrentLinkI, PreviousLink, VecOp, CondOp);
+ VPReductionRecipe *RedRecipe =
+ new VPReductionRecipe(RdxDesc, CurrentLinkI, PreviousLink, VecOp,
+ CondOp, CM.useOrderedReductions(RdxDesc));
// Append the recipe to the end of the VPBasicBlock because we need to
// ensure that it comes after all of it's inputs, including CondOp.
// Note that this transformation may leave over dead recipes (including
@@ -9307,57 +9299,6 @@
NeedsMaskForGaps);
}
-void VPReductionRecipe::execute(VPTransformState &State) {
- assert(!State.Instance && "Reduction being replicated.");
- Value *PrevInChain = State.get(getChainOp(), 0, /*IsScalar*/ true);
- RecurKind Kind = RdxDesc.getRecurrenceKind();
- bool IsOrdered = State.ILV->useOrderedReductions(RdxDesc);
- // Propagate the fast-math flags carried by the underlying instruction.
- IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder);
- State.Builder.setFastMathFlags(RdxDesc.getFastMathFlags());
- for (unsigned Part = 0; Part < State.UF; ++Part) {
- Value *NewVecOp = State.get(getVecOp(), Part);
- if (VPValue *Cond = getCondOp()) {
- Value *NewCond = State.get(Cond, Part, State.VF.isScalar());
- VectorType *VecTy = dyn_cast<VectorType>(NewVecOp->getType());
- Type *ElementTy = VecTy ? VecTy->getElementType() : NewVecOp->getType();
- Value *Iden = RdxDesc.getRecurrenceIdentity(Kind, ElementTy,
- RdxDesc.getFastMathFlags());
- if (State.VF.isVector()) {
- Iden =
- State.Builder.CreateVectorSplat(VecTy->getElementCount(), Iden);
- }
-
- Value *Select = State.Builder.CreateSelect(NewCond, NewVecOp, Iden);
- NewVecOp = Select;
- }
- Value *NewRed;
- Value *NextInChain;
- if (IsOrdered) {
- if (State.VF.isVector())
- NewRed = createOrderedReduction(State.Builder, RdxDesc, NewVecOp,
- PrevInChain);
- else
- NewRed = State.Builder.CreateBinOp(
- (Instruction::BinaryOps)RdxDesc.getOpcode(Kind), PrevInChain,
- NewVecOp);
- PrevInChain = NewRed;
- } else {
- PrevInChain = State.get(getChainOp(), Part, /*IsScalar*/ true);
- NewRed = createTargetReduction(State.Builder, RdxDesc, NewVecOp);
- }
- if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
- NextInChain = createMinMaxOp(State.Builder, RdxDesc.getRecurrenceKind(),
- NewRed, PrevInChain);
- } else if (IsOrdered)
- NextInChain = NewRed;
- else
- NextInChain = State.Builder.CreateBinOp(
- (Instruction::BinaryOps)RdxDesc.getOpcode(Kind), NewRed, PrevInChain);
- State.set(this, NextInChain, Part, /*IsScalar*/ true);
- }
-}
-
void VPReplicateRecipe::execute(VPTransformState &State) {
Instruction *UI = getUnderlyingInstr();
if (State.Instance) { // Generate a single instance.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 77577b5..5f6334b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2075,13 +2075,15 @@
class VPReductionRecipe : public VPSingleDefRecipe {
/// The recurrence decriptor for the reduction in question.
const RecurrenceDescriptor &RdxDesc;
+ bool IsOrdered;
public:
VPReductionRecipe(const RecurrenceDescriptor &R, Instruction *I,
- VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp)
+ VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
+ bool IsOrdered)
: VPSingleDefRecipe(VPDef::VPReductionSC,
ArrayRef<VPValue *>({ChainOp, VecOp}), I),
- RdxDesc(R) {
+ RdxDesc(R), IsOrdered(IsOrdered) {
if (CondOp)
addOperand(CondOp);
}
@@ -2090,7 +2092,7 @@
VPRecipeBase *clone() override {
return new VPReductionRecipe(RdxDesc, getUnderlyingInstr(), getChainOp(),
- getVecOp(), getCondOp());
+ getVecOp(), getCondOp(), IsOrdered);
}
VP_CLASSOF_IMPL(VPDef::VPReductionSC)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index fc83234..2438e4d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1518,7 +1518,58 @@
}
}
}
+#endif
+void VPReductionRecipe::execute(VPTransformState &State) {
+ assert(!State.Instance && "Reduction being replicated.");
+ Value *PrevInChain = State.get(getChainOp(), 0, /*IsScalar*/ true);
+ RecurKind Kind = RdxDesc.getRecurrenceKind();
+ // Propagate the fast-math flags carried by the underlying instruction.
+ IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder);
+ State.Builder.setFastMathFlags(RdxDesc.getFastMathFlags());
+ for (unsigned Part = 0; Part < State.UF; ++Part) {
+ Value *NewVecOp = State.get(getVecOp(), Part);
+ if (VPValue *Cond = getCondOp()) {
+ Value *NewCond = State.get(Cond, Part, State.VF.isScalar());
+ VectorType *VecTy = dyn_cast<VectorType>(NewVecOp->getType());
+ Type *ElementTy = VecTy ? VecTy->getElementType() : NewVecOp->getType();
+ Value *Iden = RdxDesc.getRecurrenceIdentity(Kind, ElementTy,
+ RdxDesc.getFastMathFlags());
+ if (State.VF.isVector()) {
+ Iden = State.Builder.CreateVectorSplat(VecTy->getElementCount(), Iden);
+ }
+
+ Value *Select = State.Builder.CreateSelect(NewCond, NewVecOp, Iden);
+ NewVecOp = Select;
+ }
+ Value *NewRed;
+ Value *NextInChain;
+ if (IsOrdered) {
+ if (State.VF.isVector())
+ NewRed = createOrderedReduction(State.Builder, RdxDesc, NewVecOp,
+ PrevInChain);
+ else
+ NewRed = State.Builder.CreateBinOp(
+ (Instruction::BinaryOps)RdxDesc.getOpcode(Kind), PrevInChain,
+ NewVecOp);
+ PrevInChain = NewRed;
+ } else {
+ PrevInChain = State.get(getChainOp(), Part, /*IsScalar*/ true);
+ NewRed = createTargetReduction(State.Builder, RdxDesc, NewVecOp);
+ }
+ if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
+ NextInChain = createMinMaxOp(State.Builder, RdxDesc.getRecurrenceKind(),
+ NewRed, PrevInChain);
+ } else if (IsOrdered)
+ NextInChain = NewRed;
+ else
+ NextInChain = State.Builder.CreateBinOp(
+ (Instruction::BinaryOps)RdxDesc.getOpcode(Kind), NewRed, PrevInChain);
+ State.set(this, NextInChain, Part, /*IsScalar*/ true);
+ }
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPReductionRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "REDUCE ";
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 02e7ca3..78d8092 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1119,7 +1119,7 @@
VPValue VecOp;
VPValue CondOp;
VPReductionRecipe Recipe(RecurrenceDescriptor(), nullptr, &ChainOp, &CondOp,
- &VecOp);
+ &VecOp, false);
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1287,7 +1287,7 @@
VPValue VecOp;
VPValue CondOp;
VPReductionRecipe Recipe(RecurrenceDescriptor(), nullptr, &ChainOp, &CondOp,
- &VecOp);
+ &VecOp, false);
EXPECT_TRUE(isa<VPUser>(&Recipe));
VPRecipeBase *BaseR = &Recipe;
EXPECT_TRUE(isa<VPUser>(BaseR));
