[VPlan] Manage scalarized values using VPValues.
This patch updates codegen to use VPValues to manage the generated
scalarized instructions.
Reviewed By: gilr
Differential Revision: https://reviews.llvm.org/D92285
GitOrigin-RevId: 54a14c264a245ae31e40581ab21be2ca5b6b1962
diff --git a/lib/Transforms/Vectorize/LoopVectorize.cpp b/lib/Transforms/Vectorize/LoopVectorize.cpp
index db7cb01..baaac8f 100644
--- a/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -514,14 +514,15 @@
/// variable canonicalization. It supports both VF = 1 for unrolled loops and
/// arbitrary length vectors.
void widenPHIInstruction(Instruction *PN, RecurrenceDescriptor *RdxDesc,
- Value *StartV, unsigned UF, ElementCount VF);
+ Value *StartV, VPValue *Def,
+ VPTransformState &State);
/// A helper function to scalarize a single Instruction in the innermost loop.
/// Generates a sequence of scalar instances for each lane between \p MinLane
/// and \p MaxLane, times each part between \p MinPart and \p MaxPart,
/// inclusive. Uses the VPValue operands from \p Operands instead of \p
/// Instr's operands.
- void scalarizeInstruction(Instruction *Instr, VPUser &Operands,
+ void scalarizeInstruction(Instruction *Instr, VPValue *Def, VPUser &Operands,
const VPIteration &Instance, bool IfPredicateInstr,
VPTransformState &State);
@@ -2903,7 +2904,8 @@
}
}
-void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr, VPUser &User,
+void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr, VPValue *Def,
+ VPUser &User,
const VPIteration &Instance,
bool IfPredicateInstr,
VPTransformState &State) {
@@ -2924,6 +2926,8 @@
if (!IsVoidRetTy)
Cloned->setName(Instr->getName() + ".cloned");
+ State.Builder.SetInsertPoint(Builder.GetInsertBlock(),
+ Builder.GetInsertPoint());
// Replace the operands of the cloned instructions with their scalar
// equivalents in the new loop.
for (unsigned op = 0, e = User.getNumOperands(); op != e; ++op) {
@@ -2940,10 +2944,7 @@
// Place the cloned scalar in the new loop.
Builder.Insert(Cloned);
- // TODO: Set result for VPValue of VPReciplicateRecipe. This requires
- // representing scalar values in VPTransformState. Add the cloned scalar to
- // the scalar map entry.
- VectorLoopValueMap.setScalarValue(Instr, Instance, Cloned);
+ State.set(Def, Instr, Cloned, Instance);
// If we just cloned a new assumption, add it the assumption cache.
if (auto *II = dyn_cast<IntrinsicInst>(Cloned))
@@ -4623,19 +4624,20 @@
void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
RecurrenceDescriptor *RdxDesc,
- Value *StartV, unsigned UF,
- ElementCount VF) {
- assert(!VF.isScalable() && "scalable vectors not yet supported.");
+ Value *StartV, VPValue *Def,
+ VPTransformState &State) {
+ assert(!State.VF.isScalable() && "scalable vectors not yet supported.");
PHINode *P = cast<PHINode>(PN);
if (EnableVPlanNativePath) {
// Currently we enter here in the VPlan-native path for non-induction
// PHIs where all control flow is uniform. We simply widen these PHIs.
// Create a vector phi with no operands - the vector phi operands will be
// set at the end of vector code generation.
- Type *VecTy =
- (VF.isScalar()) ? PN->getType() : VectorType::get(PN->getType(), VF);
+ Type *VecTy = (State.VF.isScalar())
+ ? PN->getType()
+ : VectorType::get(PN->getType(), State.VF);
Value *VecPhi = Builder.CreatePHI(VecTy, PN->getNumOperands(), "vec.phi");
- VectorLoopValueMap.setVectorValue(P, 0, VecPhi);
+ State.set(Def, P, VecPhi, 0);
OrigPHIsToFix.push_back(P);
return;
@@ -4651,9 +4653,9 @@
if (RdxDesc || Legal->isFirstOrderRecurrence(P)) {
Value *Iden = nullptr;
bool ScalarPHI =
- (VF.isScalar()) || Cost->isInLoopReduction(cast<PHINode>(PN));
+ (State.VF.isScalar()) || Cost->isInLoopReduction(cast<PHINode>(PN));
Type *VecTy =
- ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), VF);
+ ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
if (RdxDesc) {
assert(Legal->isReductionVariable(P) && StartV &&
@@ -4667,7 +4669,8 @@
} else {
IRBuilderBase::InsertPointGuard IPBuilder(Builder);
Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
- StartV = Iden = Builder.CreateVectorSplat(VF, StartV, "minmax.ident");
+ StartV = Iden =
+ Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
}
} else {
Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity(
@@ -4675,7 +4678,7 @@
Iden = IdenC;
if (!ScalarPHI) {
- Iden = ConstantVector::getSplat(VF, IdenC);
+ Iden = ConstantVector::getSplat(State.VF, IdenC);
IRBuilderBase::InsertPointGuard IPBuilder(Builder);
Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
Constant *Zero = Builder.getInt32(0);
@@ -4684,11 +4687,11 @@
}
}
- for (unsigned Part = 0; Part < UF; ++Part) {
+ for (unsigned Part = 0; Part < State.UF; ++Part) {
// This is phase one of vectorizing PHIs.
Value *EntryPart = PHINode::Create(
VecTy, 2, "vec.phi", &*LoopVectorBody->getFirstInsertionPt());
- VectorLoopValueMap.setVectorValue(P, Part, EntryPart);
+ State.set(Def, P, EntryPart, Part);
if (StartV) {
// Make sure to add the reduction start value only to the
// first unroll part.
@@ -4723,25 +4726,25 @@
// Handle the pointer induction variable case.
assert(P->getType()->isPointerTy() && "Unexpected type.");
- if (Cost->isScalarAfterVectorization(P, VF)) {
+ if (Cost->isScalarAfterVectorization(P, State.VF)) {
// This is the normalized GEP that starts counting at zero.
Value *PtrInd =
Builder.CreateSExtOrTrunc(Induction, II.getStep()->getType());
// Determine the number of scalars we need to generate for each unroll
// iteration. If the instruction is uniform, we only need to generate the
// first lane. Otherwise, we generate all VF values.
- unsigned Lanes =
- Cost->isUniformAfterVectorization(P, VF) ? 1 : VF.getKnownMinValue();
+ unsigned Lanes = Cost->isUniformAfterVectorization(P, State.VF)
+ ? 1
+ : State.VF.getKnownMinValue();
for (unsigned Part = 0; Part < UF; ++Part) {
for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
- Constant *Idx = ConstantInt::get(PtrInd->getType(),
- Lane + Part * VF.getKnownMinValue());
+ Constant *Idx = ConstantInt::get(
+ PtrInd->getType(), Lane + Part * State.VF.getKnownMinValue());
Value *GlobalIdx = Builder.CreateAdd(PtrInd, Idx);
Value *SclrGep =
emitTransformedIndex(Builder, GlobalIdx, PSE.getSE(), DL, II);
SclrGep->setName("next.gep");
- VectorLoopValueMap.setScalarValue(P, VPIteration(Part, Lane),
- SclrGep);
+ State.set(Def, P, SclrGep, VPIteration(Part, Lane));
}
}
return;
@@ -4768,28 +4771,28 @@
ScStValueType->getPointerElementType(), NewPointerPhi,
Builder.CreateMul(
ScalarStepValue,
- ConstantInt::get(PhiType, VF.getKnownMinValue() * UF)),
+ ConstantInt::get(PhiType, State.VF.getKnownMinValue() * State.UF)),
"ptr.ind", InductionLoc);
NewPointerPhi->addIncoming(InductionGEP, LoopLatch);
// Create UF many actual address geps that use the pointer
// phi as base and a vectorized version of the step value
// (<step*0, ..., step*N>) as offset.
- for (unsigned Part = 0; Part < UF; ++Part) {
+ for (unsigned Part = 0; Part < State.UF; ++Part) {
SmallVector<Constant *, 8> Indices;
// Create a vector of consecutive numbers from zero to VF.
- for (unsigned i = 0; i < VF.getKnownMinValue(); ++i)
+ for (unsigned i = 0; i < State.VF.getKnownMinValue(); ++i)
Indices.push_back(
- ConstantInt::get(PhiType, i + Part * VF.getKnownMinValue()));
+ ConstantInt::get(PhiType, i + Part * State.VF.getKnownMinValue()));
Constant *StartOffset = ConstantVector::get(Indices);
Value *GEP = Builder.CreateGEP(
ScStValueType->getPointerElementType(), NewPointerPhi,
- Builder.CreateMul(
- StartOffset,
- Builder.CreateVectorSplat(VF.getKnownMinValue(), ScalarStepValue),
- "vector.gep"));
- VectorLoopValueMap.setVectorValue(P, Part, GEP);
+ Builder.CreateMul(StartOffset,
+ Builder.CreateVectorSplat(
+ State.VF.getKnownMinValue(), ScalarStepValue),
+ "vector.gep"));
+ State.set(Def, P, GEP, Part);
}
}
}
@@ -8588,6 +8591,10 @@
auto *PHIRecipe = Instr->getType()->isVoidTy()
? nullptr
: new VPPredInstPHIRecipe(Plan->getOrAddVPValue(Instr));
+ if (PHIRecipe) {
+ Plan->removeVPValueFor(Instr);
+ Plan->addVPValue(Instr, PHIRecipe);
+ }
auto *Exit = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe);
auto *Pred = new VPBasicBlock(Twine(RegionName) + ".if", PredRecipe);
VPRegionBlock *Region = new VPRegionBlock(Entry, Exit, RegionName, true);
@@ -9044,7 +9051,8 @@
void VPWidenPHIRecipe::execute(VPTransformState &State) {
Value *StartV =
getStartValue() ? getStartValue()->getLiveInIRValue() : nullptr;
- State.ILV->widenPHIInstruction(Phi, RdxDesc, StartV, State.UF, State.VF);
+ State.ILV->widenPHIInstruction(cast<PHINode>(getUnderlyingValue()), RdxDesc,
+ StartV, this, State);
}
void VPBlendRecipe::execute(VPTransformState &State) {
@@ -9083,7 +9091,7 @@
}
}
for (unsigned Part = 0; Part < State.UF; ++Part)
- State.ValueMap.setVectorValue(Phi, Part, Entry[Part]);
+ State.set(this, Phi, Entry[Part], Part);
}
void VPInterleaveRecipe::execute(VPTransformState &State) {
@@ -9127,7 +9135,7 @@
void VPReplicateRecipe::execute(VPTransformState &State) {
if (State.Instance) { // Generate a single instance.
assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
- State.ILV->scalarizeInstruction(getUnderlyingInstr(), *this,
+ State.ILV->scalarizeInstruction(getUnderlyingInstr(), this, *this,
*State.Instance, IsPredicated, State);
// Insert scalar instance packing it into a vector.
if (AlsoPack && State.VF.isVector()) {
@@ -9136,11 +9144,9 @@
assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
Value *Poison = PoisonValue::get(
VectorType::get(getUnderlyingValue()->getType(), State.VF));
- State.ValueMap.setVectorValue(getUnderlyingInstr(),
- State.Instance->Part, Poison);
+ State.set(this, getUnderlyingInstr(), Poison, State.Instance->Part);
}
- State.ILV->packScalarIntoVectorValue(getUnderlyingInstr(),
- *State.Instance);
+ State.ILV->packScalarIntoVectorValue(this, *State.Instance, State);
}
return;
}
@@ -9153,7 +9159,7 @@
"Can't scalarize a scalable vector");
for (unsigned Part = 0; Part < State.UF; ++Part)
for (unsigned Lane = 0; Lane < EndLane; ++Lane)
- State.ILV->scalarizeInstruction(getUnderlyingInstr(), *this,
+ State.ILV->scalarizeInstruction(getUnderlyingInstr(), this, *this,
VPIteration(Part, Lane), IsPredicated,
State);
}
@@ -9191,6 +9197,8 @@
BasicBlock *PredicatedBB = ScalarPredInst->getParent();
BasicBlock *PredicatingBB = PredicatedBB->getSinglePredecessor();
assert(PredicatingBB && "Predicated block has no single predecessor.");
+ assert(isa<VPReplicateRecipe>(getOperand(0)) &&
+ "operand must be VPReplicateRecipe");
// By current pack/unpack logic we need to generate only a single phi node: if
// a vector value for the predicated instruction exists at this point it means
@@ -9199,21 +9207,32 @@
// also do that packing, thereby "hoisting" the insert-element sequence.
// Otherwise, a phi node for the scalar value is needed.
unsigned Part = State.Instance->Part;
- Instruction *PredInst =
- cast<Instruction>(getOperand(0)->getUnderlyingValue());
- if (State.ValueMap.hasVectorValue(PredInst, Part)) {
- Value *VectorValue = State.ValueMap.getVectorValue(PredInst, Part);
+ if (State.hasVectorValue(getOperand(0), Part)) {
+ Value *VectorValue = State.get(getOperand(0), Part);
InsertElementInst *IEI = cast<InsertElementInst>(VectorValue);
PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2);
VPhi->addIncoming(IEI->getOperand(0), PredicatingBB); // Unmodified vector.
VPhi->addIncoming(IEI, PredicatedBB); // New vector with inserted element.
- State.ValueMap.resetVectorValue(PredInst, Part, VPhi); // Update cache.
+ if (State.hasVectorValue(this, Part))
+ State.reset(this, VPhi, Part);
+ else
+ State.set(this, VPhi, Part);
+ // NOTE: Currently we need to update the value of the operand, so the next
+ // predicated iteration inserts its generated value in the correct vector.
+ State.reset(getOperand(0), VPhi, Part);
} else {
- Type *PredInstType = PredInst->getType();
+ Type *PredInstType = getOperand(0)->getUnderlyingValue()->getType();
PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);
- Phi->addIncoming(PoisonValue::get(ScalarPredInst->getType()), PredicatingBB);
+ Phi->addIncoming(PoisonValue::get(ScalarPredInst->getType()),
+ PredicatingBB);
Phi->addIncoming(ScalarPredInst, PredicatedBB);
- State.ValueMap.resetScalarValue(PredInst, *State.Instance, Phi);
+ if (State.hasScalarValue(this, *State.Instance))
+ State.reset(this, Phi, *State.Instance);
+ else
+ State.set(this, Phi, *State.Instance);
+ // NOTE: Currently we need to update the value of the operand, so the next
+ // predicated iteration inserts its generated value in the correct vector.
+ State.reset(getOperand(0), Phi, *State.Instance);
}
}
@@ -9297,8 +9316,6 @@
if (hasVectorValue(Def, Part))
return Data.PerPartOutput[Def][Part];
- // TODO: Remove the callback once all scalar recipes are managed using
- // VPValues.
if (!hasScalarValue(Def, {Part, 0}))
return Callback.getOrCreateVectorValues(VPValue2Value[Def], Part);
@@ -9336,7 +9353,7 @@
} else {
// Initialize packing with insertelements to start from undef.
assert(!VF.isScalable() && "VF is assumed to be non scalable.");
- Value *Undef = UndefValue::get(VectorType::get(LastInst->getType(), VF));
+ Value *Undef = PoisonValue::get(VectorType::get(LastInst->getType(), VF));
set(Def, Undef, Part);
for (unsigned Lane = 0; Lane < VF.getKnownMinValue(); ++Lane)
ILV->packScalarIntoVectorValue(Def, {Part, Lane}, *this);
diff --git a/lib/Transforms/Vectorize/VPlan.cpp b/lib/Transforms/Vectorize/VPlan.cpp
index 63ffaed..942208d 100644
--- a/lib/Transforms/Vectorize/VPlan.cpp
+++ b/lib/Transforms/Vectorize/VPlan.cpp
@@ -195,18 +195,17 @@
if (hasScalarValue(Def, Instance))
return Data.PerPartScalars[Def][Instance.Part][Instance.Lane];
- if (hasVectorValue(Def, Instance.Part)) {
- assert(Data.PerPartOutput.count(Def));
- auto *VecPart = Data.PerPartOutput[Def][Instance.Part];
- if (!VecPart->getType()->isVectorTy()) {
- assert(Instance.Lane == 0 && "cannot get lane > 0 for scalar");
- return VecPart;
- }
- // TODO: Cache created scalar values.
- return Builder.CreateExtractElement(VecPart,
- Builder.getInt32(Instance.Lane));
+ assert(hasVectorValue(Def, Instance.Part));
+ auto *VecPart = Data.PerPartOutput[Def][Instance.Part];
+ if (!VecPart->getType()->isVectorTy()) {
+ assert(Instance.Lane == 0 && "cannot get lane > 0 for scalar");
+ return VecPart;
}
- return Callback.getOrCreateScalarValue(VPValue2Value[Def], Instance);
+ // TODO: Cache created scalar values.
+ auto *Extract =
+ Builder.CreateExtractElement(VecPart, Builder.getInt32(Instance.Lane));
+ // set(Def, Extract, Instance);
+ return Extract;
}
BasicBlock *
@@ -884,7 +883,7 @@
void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
- O << "WIDEN-PHI " << VPlanIngredient(Phi);
+ O << "WIDEN-PHI " << VPlanIngredient(getUnderlyingValue());
}
void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent,
@@ -941,6 +940,8 @@
void VPPredInstPHIRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << "PHI-PREDICATED-INSTRUCTION ";
+ printAsOperand(O, SlotTracker);
+ O << " = ";
printOperands(O, SlotTracker);
}
diff --git a/lib/Transforms/Vectorize/VPlan.h b/lib/Transforms/Vectorize/VPlan.h
index a38cd08..9b93331 100644
--- a/lib/Transforms/Vectorize/VPlan.h
+++ b/lib/Transforms/Vectorize/VPlan.h
@@ -311,10 +311,21 @@
}
Data.PerPartOutput[Def][Part] = V;
}
+ /// Reset an existing vector value for \p Def and a given \p Part.
+ void reset(VPValue *Def, Value *V, unsigned Part) {
+ auto Iter = Data.PerPartOutput.find(Def);
+ assert(Iter != Data.PerPartOutput.end() &&
+ "need to overwrite existing value");
+ Iter->second[Part] = V;
+ }
+
void set(VPValue *Def, Value *IRDef, Value *V, unsigned Part);
void reset(VPValue *Def, Value *IRDef, Value *V, unsigned Part);
- void set(VPValue *Def, Value *IRDef, Value *V, const VPIteration &Instance);
+ /// Set the generated scalar \p V for \p Def and \p IRDef and the given \p
+ /// Instance.
+ void set(VPValue *Def, Value *IRDef, Value *V, const VPIteration &Instance);
+ /// Set the generated scalar \p V for \p Def and the given \p Instance.
void set(VPValue *Def, Value *V, const VPIteration &Instance) {
auto Iter = Data.PerPartScalars.insert({Def, {}});
auto &PerPartVec = Iter.first->second;
@@ -323,9 +334,22 @@
auto &Scalars = PerPartVec[Instance.Part];
while (Scalars.size() <= Instance.Lane)
Scalars.push_back(nullptr);
+ assert(!Scalars[Instance.Lane] && "should overwrite existing value");
Scalars[Instance.Lane] = V;
}
+ /// Reset an existing scalar value for \p Def and a given \p Instance.
+ void reset(VPValue *Def, Value *V, const VPIteration &Instance) {
+ auto Iter = Data.PerPartScalars.find(Def);
+ assert(Iter != Data.PerPartScalars.end() &&
+ "need to overwrite existing value");
+ assert(Instance.Part < Iter->second.size() &&
+ "need to overwrite existing value");
+ assert(Instance.Lane < Iter->second[Instance.Part].size() &&
+ "need to overwrite existing value");
+ Iter->second[Instance.Part][Instance.Lane] = V;
+ }
+
/// Hold state information used when constructing the CFG of the output IR,
/// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
struct CFGState {
@@ -1016,7 +1040,7 @@
/// A recipe for handling all phi nodes except for integer and FP inductions.
/// For reduction PHIs, RdxDesc must point to the corresponding recurrence
/// descriptor and the start value is the first operand of the recipe.
-class VPWidenPHIRecipe : public VPRecipeBase {
+class VPWidenPHIRecipe : public VPRecipeBase, public VPValue {
PHINode *Phi;
/// Descriptor for a reduction PHI.
@@ -1032,10 +1056,9 @@
}
/// Create a VPWidenPHIRecipe for \p Phi
- VPWidenPHIRecipe(PHINode *Phi) : VPRecipeBase(VPWidenPHISC, {}), Phi(Phi) {
- new VPValue(Phi, this);
- }
-
+ VPWidenPHIRecipe(PHINode *Phi)
+ : VPRecipeBase(VPWidenPHISC, {}),
+ VPValue(VPValue::VPVWidenPHISC, Phi, this), Phi(Phi) {}
~VPWidenPHIRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast.
@@ -1058,7 +1081,7 @@
/// A recipe for vectorizing a phi-node as a sequence of mask-based select
/// instructions.
-class VPBlendRecipe : public VPRecipeBase {
+class VPBlendRecipe : public VPRecipeBase, public VPValue {
PHINode *Phi;
public:
@@ -1066,8 +1089,8 @@
/// respective masks, ordered [I0, M0, I1, M1, ...]. Note that a single value
/// might be incoming with a full mask for which there is no VPValue.
VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands)
- : VPRecipeBase(VPBlendSC, Operands), Phi(Phi) {
- new VPValue(Phi, this);
+ : VPRecipeBase(VPBlendSC, Operands),
+ VPValue(VPValue::VPVBlendSC, Phi, this), Phi(Phi) {
assert(Operands.size() > 0 &&
((Operands.size() == 1) || (Operands.size() % 2 == 0)) &&
"Expected either a single incoming value or a positive even number "
@@ -1260,6 +1283,8 @@
VPSlotTracker &SlotTracker) const override;
bool isUniform() const { return IsUniform; }
+
+ bool isPacked() const { return AlsoPack; }
};
/// A recipe for generating conditional branches on the bits of a mask.
@@ -1305,14 +1330,13 @@
/// order to merge values that are set under such a branch and feed their uses.
/// The phi nodes can be scalar or vector depending on the users of the value.
/// This recipe works in concert with VPBranchOnMaskRecipe.
-class VPPredInstPHIRecipe : public VPRecipeBase {
-
+class VPPredInstPHIRecipe : public VPRecipeBase, public VPValue {
public:
/// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
/// nodes after merging back from a Branch-on-Mask.
- VPPredInstPHIRecipe(VPValue *PredV) : VPRecipeBase(VPPredInstPHISC, PredV) {
- new VPValue(PredV->getUnderlyingValue(), this);
- }
+ VPPredInstPHIRecipe(VPValue *PredV)
+ : VPRecipeBase(VPPredInstPHISC, PredV),
+ VPValue(VPValue::VPVPredInstPHI, nullptr, this) {}
~VPPredInstPHIRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast.
diff --git a/lib/Transforms/Vectorize/VPlanTransforms.cpp b/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 8b569fb..ead84ff 100644
--- a/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -75,6 +75,10 @@
assert(NewRecipe->getNumDefinedValues() == 0 &&
"Only recpies with zero or one defined values expected");
Ingredient->eraseFromParent();
+ Plan->removeVPValueFor(Inst);
+ for (auto *Def : NewRecipe->definedValues()) {
+ Plan->addVPValue(Inst, Def);
+ }
}
}
}
diff --git a/lib/Transforms/Vectorize/VPlanValue.h b/lib/Transforms/Vectorize/VPlanValue.h
index c4773eb..55c2c74 100644
--- a/lib/Transforms/Vectorize/VPlanValue.h
+++ b/lib/Transforms/Vectorize/VPlanValue.h
@@ -90,13 +90,17 @@
/// type identification.
enum {
VPValueSC,
+ VPVBlendSC,
VPVInstructionSC,
VPVMemoryInstructionSC,
+ VPVPredInstPHI,
VPVReductionSC,
VPVReplicateSC,
VPVWidenSC,
VPVWidenCallSC,
VPVWidenGEPSC,
+ VPVWidenIntOrFpIndcutionSC,
+ VPVWidenPHISC,
VPVWidenSelectSC,
};
diff --git a/test/Transforms/LoopVectorize/vplan-printing.ll b/test/Transforms/LoopVectorize/vplan-printing.ll
index e3b9f33..1f649f3 100644
--- a/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -118,13 +118,13 @@
;
; CHECK: N5 [label =
; CHECK-NEXT: "pred.udiv.continue:\n" +
-; CHECK-NEXT: "PHI-PREDICATED-INSTRUCTION ir<%tmp4>\l"
+; CHECK-NEXT: "PHI-PREDICATED-INSTRUCTION vp<%3> = ir<%tmp4>\l"
; CHECK-NEXT: ]
;
; CHECK: N7 [label =
; CHECK-NEXT: "for.inc:\n" +
; CHECK-NEXT: "EMIT vp<%4> = not ir<%cmp>\l" +
-; CHECK-NEXT: "BLEND %d = ir<0>/vp<%4> ir<%tmp4>/ir<%cmp>\l" +
+; CHECK-NEXT: "BLEND %d = ir<0>/vp<%4> vp<%3>/ir<%cmp>\l" +
; CHECK-NEXT: "CLONE ir<%idx> = getelementptr ir<%x>, ir<%i>\l" +
; CHECK-NEXT: "WIDEN store ir<%idx>, ir<%d>\l"
; CHECK-NEXT: ]
