[review-suggestions] Replace VPMonotonicPHIRecipe with VPPhi
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 4762e05..e1fd159 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4084,7 +4084,6 @@ case VPDef::VPEVLBasedIVPHISC: case VPDef::VPPredInstPHISC: case VPDef::VPBranchOnMaskSC: - case VPDef::VPMonotonicPHISC: continue; case VPDef::VPReductionSC: case VPDef::VPActiveLaneMaskPHISC: @@ -8115,7 +8114,7 @@ if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, Range))) return Recipe; - VPHeaderPHIRecipe *PhiRecipe = nullptr; + VPSingleDefRecipe *PhiRecipe = nullptr; assert((Legal->isMonotonicPHI(Phi) || Legal->isReductionVariable(Phi) || Legal->isFixedOrderRecurrence(Phi)) && "can only widen monotonic phis, reductions and fixed-order " @@ -8124,10 +8123,9 @@ Value *IncomingVal = Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()); if (Legal->isMonotonicPHI(Phi)) { - const MonotonicDescriptor &Desc = - Legal->getMonotonicPHIs().find(Phi)->second; - assert(Desc.getExpr()->getStart() == PSE.getSCEV(IncomingVal)); - PhiRecipe = new VPMonotonicPHIRecipe(Phi, Desc, StartV); + PhiRecipe = new VPPhi({StartV}, Phi->getDebugLoc(), + Phi->getName() + ".monotonic"); + PhiRecipe->setUnderlyingValue(Phi); } else if (Legal->isReductionVariable(Phi)) { const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(Phi); assert(RdxDesc.getRecurrenceStartValue() == @@ -8480,10 +8478,19 @@ // --------------------------------------------------------------------------- // Adjust the recipes for any monotonic phis. + auto &MonotonicPHIs = Legal->getMonotonicPHIs(); for (VPRecipeBase &R : HeaderVPBB->phis()) { - auto *MonotonicPhi = dyn_cast<VPMonotonicPHIRecipe>(&R); + auto *MonotonicPhi = dyn_cast<VPPhi>(&R); if (!MonotonicPhi) continue; + assert(MonotonicPhi->getNumIncoming() == 2 && + MonotonicPhi->getIncomingBlock(0) == Plan->getVectorPreheader()); + + auto It = + MonotonicPHIs.find(cast<PHINode>(MonotonicPhi->getUnderlyingValue())); + if (It == MonotonicPHIs.end()) + continue; + auto &Desc = It->second; // Prohibit scalarization of monotonic phis. if (!all_of(Range, [&](ElementCount VF) { @@ -8494,7 +8501,7 @@ // Obtain mask value for the predicate edge from the last VPBlendRecipe in // chain. - VPValue *Chain = MonotonicPhi->getBackedgeValue(); + VPValue *Chain = MonotonicPhi->getIncomingValue(1); VPValue *Mask = nullptr; while (auto *BlendR = dyn_cast<VPBlendRecipe>(Chain)) for (unsigned I = 0, E = BlendR->getNumIncomingValues(); I != E; ++I) @@ -8506,7 +8513,6 @@ } assert(Mask); - auto &Desc = MonotonicPhi->getDescriptor(); auto &SE = *PSE.getSE(); auto *Step = vputils::getOrCreateVPValueForSCEVExpr( *Plan, Desc.getExpr()->getStepRecurrence(SE)); @@ -8514,9 +8520,10 @@ auto *MonotonicI = new VPInstruction(VPInstruction::ComputeMonotonicResult, {MonotonicPhi, Mask, Step}, *Desc.getStepInst()); - auto *InsertBlock = MonotonicPhi->getBackedgeRecipe().getParent(); + auto *BackedgeVal = MonotonicPhi->getIncomingValue(1); + auto *InsertBlock = BackedgeVal->getDefiningRecipe()->getParent(); InsertBlock->insert(MonotonicI, InsertBlock->getFirstNonPhi()); - MonotonicPhi->getBackedgeValue()->replaceAllUsesWith(MonotonicI); + BackedgeVal->replaceAllUsesWith(MonotonicI); } // Adjust the recipes for any inloop reductions.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 87fd6b5..dd26a05 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -984,7 +984,6 @@ auto *PhiR = cast<VPSingleDefRecipe>(&R); // VPInstructions currently model scalar Phis only. bool NeedsScalar = isa<VPInstruction>(PhiR) || - isa<VPMonotonicPHIRecipe>(PhiR) || (isa<VPReductionPHIRecipe>(PhiR) && cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index d442b90..a763e82 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -553,7 +553,6 @@ case VPRecipeBase::VPWidenIntOrFpInductionSC: case VPRecipeBase::VPWidenPointerInductionSC: case VPRecipeBase::VPReductionPHISC: - case VPRecipeBase::VPMonotonicPHISC: case VPRecipeBase::VPPartialReductionSC: return true; case VPRecipeBase::VPBranchOnMaskSC: @@ -2408,50 +2407,6 @@ } }; -/// A recipe for handling monotonic phis. The start value is the first operand -/// of the recipe and the incoming value from the backedge is the second -/// operand. -class VPMonotonicPHIRecipe : public VPHeaderPHIRecipe { - MonotonicDescriptor Desc; - -public: - VPMonotonicPHIRecipe(PHINode *Phi, const MonotonicDescriptor &Desc, - VPValue *Start) - : VPHeaderPHIRecipe(VPDef::VPMonotonicPHISC, Phi, Start), Desc(Desc) {} - - ~VPMonotonicPHIRecipe() override = default; - - VPMonotonicPHIRecipe *clone() override { - auto *R = new VPMonotonicPHIRecipe(cast<PHINode>(getUnderlyingInstr()), - Desc, getStartValue()); - R->addOperand(getBackedgeValue()); - return R; - } - - VP_CLASSOF_IMPL(VPDef::VPMonotonicPHISC) - - static inline bool classof(const VPHeaderPHIRecipe *R) { - return R->getVPDefID() == VPDef::VPMonotonicPHISC; - } - - void execute(VPTransformState &State) override; - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - /// Print the recipe. - void print(raw_ostream &O, const Twine &Indent, - VPSlotTracker &SlotTracker) const override; -#endif - - const MonotonicDescriptor &getDescriptor() const { return Desc; } - - /// Returns true if the recipe only uses the first lane of operand \p Op. - bool usesFirstLaneOnly(const VPValue *Op) const override { - assert(is_contained(operands(), Op) && - "Op must be an operand of the recipe"); - return true; - } -}; - /// A recipe for vectorizing a phi-node as a sequence of mask-based select /// instructions. class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPSingleDefRecipe {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp index da9e1f2..94b4d5f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -98,7 +98,7 @@ return inferScalarType(R->getOperand(0)); } case VPInstruction::ComputeMonotonicResult: { - auto *PhiR = cast<VPMonotonicPHIRecipe>(R->getOperand(0)); + auto *PhiR = cast<VPPhi>(R->getOperand(0)); auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue()); return OrigPhi->getType(); } @@ -281,14 +281,14 @@ TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe()) .Case<VPActiveLaneMaskPHIRecipe, VPCanonicalIVPHIRecipe, VPFirstOrderRecurrencePHIRecipe, VPReductionPHIRecipe, - VPMonotonicPHIRecipe, VPWidenPointerInductionRecipe, - VPEVLBasedIVPHIRecipe>([this](const auto *R) { - // Handle header phi recipes, except VPWidenIntOrFpInduction - // which needs special handling due it being possibly truncated. - // TODO: consider inferring/caching type of siblings, e.g., - // backedge value, here and in cases below. - return inferScalarType(R->getStartValue()); - }) + VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe>( + [this](const auto *R) { + // Handle header phi recipes, except VPWidenIntOrFpInduction + // which needs special handling due it being possibly truncated. + // TODO: consider inferring/caching type of siblings, e.g., + // backedge value, here and in cases below. + return inferScalarType(R->getStartValue()); + }) .Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe>( [](const auto *R) { return R->getScalarType(); }) .Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 437e5c9..132e9a0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1217,8 +1217,8 @@ default: // TODO: Compute cost other VPInstructions once the legacy cost model has // been retired. - assert(!getUnderlyingValue() && - "unexpected VPInstruction witht underlying value"); + assert((getOpcode() == Instruction::PHI || !getUnderlyingValue()) && + "unexpected VPInstruction with underlying value"); return 0; } } @@ -4495,29 +4495,6 @@ } #endif -void VPMonotonicPHIRecipe::execute(VPTransformState &State) { - assert(getParent()->getPlan()->getUF() == 1 && "Expected unroll factor 1."); - Value *Start = getStartValue()->getLiveInIRValue(); - BasicBlock *VectorPH = - State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0)); - PHINode *MonotonicPHI = - State.Builder.CreatePHI(Start->getType(), 2, "monotonic.iv"); - MonotonicPHI->addIncoming(Start, VectorPH); - MonotonicPHI->setDebugLoc(getDebugLoc()); - State.set(this, MonotonicPHI, /*IsScalar=*/true); -} - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -void VPMonotonicPHIRecipe::print(raw_ostream &O, const Twine &Indent, - VPSlotTracker &SlotTracker) const { - O << Indent << "MONOTONIC-PHI "; - - printAsOperand(O, SlotTracker); - O << " = phi "; - printOperands(O, SlotTracker); -} -#endif - void VPWidenPHIRecipe::execute(VPTransformState &State) { Value *Op0 = State.get(getOperand(0)); Type *VecTy = Op0->getType();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index cdfbc53..bc305f5 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4448,8 +4448,7 @@ // TODO: Extract final value from induction recipe initially, optimize to // pre-computed end value together in optimizeInductionExitUsers. - auto *VectorPhiR = - cast<VPHeaderPHIRecipe>(Builder.getRecipe(&ScalarPhiIRI->getIRPhi())); + VPRecipeBase *VectorPhiR = Builder.getRecipe(&ScalarPhiIRI->getIRPhi()); if (auto *WideIVR = dyn_cast<VPWidenInductionRecipe>(VectorPhiR)) { if (VPInstruction *ResumePhi = addResumePhiRecipeForInduction( WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo, @@ -4471,7 +4470,8 @@ // which for FORs is a vector whose last element needs to be extracted. The // start value provides the value if the loop is bypassed. bool IsFOR = isa<VPFirstOrderRecurrencePHIRecipe>(VectorPhiR); - auto *ResumeFromVectorLoop = VectorPhiR->getBackedgeValue(); + auto *PhiAccessor = cast<VPPhiAccessors>(VectorPhiR); + auto *ResumeFromVectorLoop = PhiAccessor->getIncomingValue(1); assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() && "Cannot handle loops with uncountable early exits"); if (IsFOR) @@ -4480,7 +4480,7 @@ "vector.recur.extract"); StringRef Name = IsFOR ? "scalar.recur.init" : "bc.merge.rdx"; auto *ResumePhiR = ScalarPHBuilder.createScalarPhi( - {ResumeFromVectorLoop, VectorPhiR->getStartValue()}, {}, Name); + {ResumeFromVectorLoop, PhiAccessor->getIncomingValue(0)}, {}, Name); ScalarPhiIRI->addOperand(ResumePhiR); } }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index 4fd2398..5da7463 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -371,13 +371,12 @@ VPWidenIntOrFpInductionSC, VPWidenPointerInductionSC, VPReductionPHISC, - VPMonotonicPHISC, // END: SubclassID for recipes that inherit VPHeaderPHIRecipe // END: Phi-like recipes VPFirstPHISC = VPWidenPHISC, VPFirstHeaderPHISC = VPCanonicalIVPHISC, - VPLastHeaderPHISC = VPMonotonicPHISC, - VPLastPHISC = VPMonotonicPHISC, + VPLastHeaderPHISC = VPReductionPHISC, + VPLastPHISC = VPReductionPHISC, }; VPDef(const unsigned char SC) : SubclassID(SC) {}