[VPlan] Transform reverse loads into vp.strided.load with -1 stride Rebased from the WIP on loop-vectorize/reverse-vp-strided-load. After simplifyReverses pulls reverses out of elementwise recipes, look through single-use elementwise trees under a reverse to the reverse consecutive loads at their leaves (live-ins are allowed as splats), and replace each load with a vp.strided.load with a negative stride when it is legal and cheaper than the loads plus the reverse. This is the VPlan equivalent of RISC-V's performReverseEVLCombine. Fixes #204154.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index a04931c..5e2a998 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6309,6 +6309,19 @@ RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan, Config.getMinimalBitwidths()); RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan); + // This runs after optimize since it relies on simplifyReverses to pull + // reverses out of elementwise recipes. If it clamps the range, drop the + // VFs past the new end so the next iteration builds a plan for them. + if (!Plan->hasScalarVFOnly()) { + VPCostContext CostCtx(*TLI, *Plan, PlanCM, Config); + RUN_VPLAN_PASS(VPlanTransforms::convertToReverseStridedAccess, *Plan, + CostCtx, SubRange); + for (ElementCount RemovedVF = SubRange.End; + ElementCount::isKnownLT(RemovedVF, VF); RemovedVF *= 2) + Plan->removeVF(RemovedVF); + VF = SubRange.End; + } + // TODO: try to put addExplicitVectorLength close to addActiveLaneMask if (PlanCM.foldTailWithEVL()) { RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index f5558ed..bb34c83 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1811,6 +1811,127 @@ R.getVPSingleValue()->replaceAllUsesWith(X); } +/// If \p R is a reverse of a tree of single-use elementwise recipes whose +/// leaves are single scalars or consecutive loads, replace each load with a +/// vp.strided.load with a negative stride that starts at the highest address, +/// and drop the reverse: +/// +/// reverse(op(load(endptr(p)), reverse(m)), ...)) +/// -> op(vp.strided.load(p, -sizeof(elt), m, VF), ...) +static bool convertToReverseStridedLoads(VPRecipeBase *R, VPCostContext &Ctx, + VFRange &Range) { + VPValue *X; + if (!match(R, m_Reverse(m_VPValue(X)))) + return false; + + SmallSetVector<VPWidenLoadRecipe *, 4> Loads; + SmallVector<VPValue *> Worklist = {X}; + while (!Worklist.empty()) { + VPValue *X = Worklist.pop_back_val(); + if (vputils::isSingleScalar(X)) + continue; + auto *Def = dyn_cast_if_present<VPSingleDefRecipe>(X->getDefiningRecipe()); + if (!Def || Def->getNumUsers() != 1) + return false; + if (auto *Load = dyn_cast<VPWidenLoadRecipe>(Def); + Load && Load->isConsecutive() && + (!Load->getMask() || match(Load->getMask(), m_Reverse(m_VPValue())))) { + Loads.insert(Load); + } else if (vputils::isElementwise(Def)) { + append_range(Worklist, Def->operands()); + } else { + return false; + } + } + + if (Loads.empty()) + return false; + + Type *LoadTy = Loads[0]->getScalarType(); + Align Alignment = Loads[0]->getAlign(); + auto IsProfitable = [&](ElementCount VF) { + Type *DataTy = toVectorTy(LoadTy, VF); + if (!Ctx.TTI.isLegalStridedLoadStore(DataTy, Alignment)) + return false; + + InstructionCost CurrentCost = cast<VPInstruction>(R)->computeCost(VF, Ctx); + for (auto *Load : Loads) + CurrentCost += Load->computeCost(VF, Ctx); + + InstructionCost NewCost = 0; + for (auto *Load : Loads) + NewCost += VPWidenMemIntrinsicRecipe::computeMemIntrinsicCost( + Intrinsic::experimental_vp_strided_load, DataTy, Load->isMasked(), + Alignment, Ctx); + return NewCost < CurrentCost; + }; + + if (!LoopVectorizationPlanner::getDecisionAndClampRange(IsProfitable, Range)) + return false; + + VPlan *Plan = R->getParent()->getPlan(); + + // Get VF as i32 for the vector length operand. + VPValue *I32VF = VPBuilder(Plan->getVectorPreheader()) + .createScalarZExtOrTrunc( + &Plan->getVF(), Type::getInt32Ty(Plan->getContext()), + DebugLoc::getUnknown()); + + Type *VFTy = Plan->getVF().getScalarType(); + uint64_t EltSize = Plan->getDataLayout().getTypeStoreSize(LoadTy); + for (VPWidenLoadRecipe *Load : Loads) { + Type *IndexTy = + Plan->getDataLayout().getIndexType(Load->getAddr()->getScalarType()); + VPBuilder Builder(Load); + // The strided load starts at the highest address of each part. For a + // reverse load that is the end pointer's base stepped back by VF per part. + // Otherwise rebase the lowest address: addr + (VF-1) * size. + VPValue *NewPtr; + if (auto *EndPtr = dyn_cast<VPVectorEndPointerRecipe>(Load->getAddr())) { + NewPtr = Builder.createVectorPointer( + EndPtr->getPointer(), EndPtr->getSourceElementType(), + Plan->getConstantInt(IndexTy, EndPtr->getStride(), /*IsSigned=*/true), + EndPtr->getGEPNoWrapFlags(), EndPtr->getDebugLoc()); + } else { + VPValue *VFMinusOne = + Builder.createSub(&Plan->getVF(), Plan->getConstantInt(VFTy, 1)); + VPValue *Offset = Builder.createOverflowingOp( + Instruction::Mul, {Plan->getConstantInt(VFTy, EltSize), VFMinusOne}); + NewPtr = Builder.createPtrAdd(Load->getAddr(), Offset); + } + VPValue *StrideInBytes = + Plan->getConstantInt(IndexTy, -EltSize, /*IsSigned=*/true); + // The strided load produces lanes in reverse order, so it takes the + // unreversed mask. + VPValue *Mask = Plan->getTrue(); + if (Load->getMask()) + match(Load->getMask(), m_Reverse(m_VPValue(Mask))); + auto *StridedLoad = Builder.createWidenMemIntrinsic( + Intrinsic::experimental_vp_strided_load, + {NewPtr, StrideInBytes, Mask, I32VF}, LoadTy, Alignment, *Load, + Load->getDebugLoc()); + Load->replaceAllUsesWith(StridedLoad); + } + + R->getVPSingleValue()->replaceAllUsesWith(R->getOperand(0)); + R->eraseFromParent(); + return true; +} + +void VPlanTransforms::convertToReverseStridedAccess(VPlan &Plan, + VPCostContext &Ctx, + VFRange &Range) { + VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion(); + bool Changed = false; + for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>( + vp_depth_first_shallow(VectorLoop->getEntry()))) + for (VPRecipeBase &R : make_early_inc_range(*VPBB)) + Changed |= convertToReverseStridedLoads(&R, Ctx, Range); + // Clean up the replaced loads and their reversed masks. + if (Changed) + removeDeadRecipes(Plan); +} + /// Reassociate (headermask && x) && y -> headermask && (x && y) to allow the /// header mask to be simplified further when tail folding, e.g. in /// optimizeEVLMasks.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index 7f69f6e..25b4a8f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -450,6 +450,12 @@ /// Cancel out redundant reverses in \p Plan, e.g. reverse(reverse(x)) -> x. static void simplifyReverses(VPlan &Plan); + /// Replace reverse(elementwise tree of reverse consecutive loads) with the + /// same tree of negative-stride vp.strided.loads when legal and profitable. + /// \p Range is clamped to the VFs where the decision holds. + static void convertToReverseStridedAccess(VPlan &Plan, VPCostContext &Ctx, + VFRange &Range); + /// Remove BranchOnCond recipes with true or false conditions together with /// removing dead edges to their successors. If \p OnlyLatches is true, only /// process loop latches. Returns true if incoming values from any phi-like
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll b/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll index 1306b9a..8195f42 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/iv-select-cmp.ll
@@ -86,15 +86,9 @@ ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]] ; CHECK-NEXT: [[TMP38:%.*]] = add nsw i64 [[OFFSET_IDX]], -1 ; CHECK-NEXT: [[TMP39:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP38]] -; CHECK-NEXT: [[TMP9:%.*]] = sub nuw nsw i64 [[TMP5]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = sub i64 0, [[TMP9]] -; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP39]], i64 [[TMP10]] -; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i64> @llvm.vp.load.nxv4i64.p0(ptr align 8 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]]) -; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i64> @llvm.experimental.vp.reverse.nxv4i64(<vscale x 4 x i64> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]]) +; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i64> @llvm.experimental.vp.strided.load.nxv4i64.p0.i64(ptr align 8 [[TMP39]], i64 -8, <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]]) ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP38]] -; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i64, ptr [[TMP6]], i64 [[TMP10]] -; CHECK-NEXT: [[VP_OP_LOAD4:%.*]] = call <vscale x 4 x i64> @llvm.vp.load.nxv4i64.p0(ptr align 8 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]]) -; CHECK-NEXT: [[TMP15:%.*]] = call <vscale x 4 x i64> @llvm.experimental.vp.reverse.nxv4i64(<vscale x 4 x i64> [[VP_OP_LOAD4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]]) +; CHECK-NEXT: [[TMP15:%.*]] = call <vscale x 4 x i64> @llvm.experimental.vp.strided.load.nxv4i64.p0.i64(ptr align 8 [[TMP6]], i64 -8, <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]]) ; CHECK-NEXT: [[TMP16:%.*]] = icmp sgt <vscale x 4 x i64> [[TMP12]], [[TMP15]] ; CHECK-NEXT: [[TMP18]] = call <vscale x 4 x i64> @llvm.vp.merge.nxv4i64(<vscale x 4 x i1> [[TMP16]], <vscale x 4 x i64> [[VEC_IND]], <vscale x 4 x i64> [[VEC_PHI]], i32 [[TMP4]]) ; CHECK-NEXT: [[TMP19]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[VEC_PHI1]], i32 [[TMP4]])
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reverse-load-scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reverse-load-scatter.ll index 3bcb81f..cb51001 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/reverse-load-scatter.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/reverse-load-scatter.ll
@@ -14,14 +14,10 @@ ; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true) ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[CURRENT_ITERATION_IV]] ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[OFFSET_IDX]] -; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP0]] to i64 -; CHECK-NEXT: [[TMP3:%.*]] = sub nuw nsw i64 [[TMP2]], 1 -; CHECK-NEXT: [[TMP4:%.*]] = sub i64 0, [[TMP3]] -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[TMP1]], i64 [[TMP4]] -; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) -; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.reverse.nxv2i64(<vscale x 2 x i64> [[VP_OP_LOAD]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP1]], i64 -8, <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DST]], <vscale x 2 x i64> [[TMP6]] ; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[TMP6]], <vscale x 2 x ptr> align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP0]] to i64 ; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP2]], [[CURRENT_ITERATION_IV]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP2]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reverse-strided-load.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reverse-strided-load.ll index 0cf5c36..5ca9363 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/reverse-strided-load.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/reverse-strided-load.ll
@@ -15,18 +15,13 @@ ; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true) ; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[INDEX]] ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]] -; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP0]] to i64 -; CHECK-NEXT: [[TMP4:%.*]] = sub nuw nsw i64 [[TMP3]], 1 -; CHECK-NEXT: [[TMP5:%.*]] = sub i64 0, [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP2]], i64 [[TMP5]] -; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP2]], i64 -8, <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[C]], i64 [[TMP1]] -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[TMP7]], i64 [[TMP5]] -; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP7]], i64 -8, <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) ; CHECK-NEXT: [[TMP9:%.*]] = add <vscale x 2 x i64> [[VP_OP_LOAD]], [[VP_OP_LOAD1]] -; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.reverse.nxv2i64(<vscale x 2 x i64> [[TMP9]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]] -; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP10]], ptr align 8 [[TMP11]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP11]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP0]] to i64 ; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP3]], [[INDEX]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP3]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 @@ -73,16 +68,12 @@ ; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true) ; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[INDEX]] ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[TMP1]] -; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP0]] to i64 -; CHECK-NEXT: [[TMP4:%.*]] = sub nuw nsw i64 [[TMP3]], 1 -; CHECK-NEXT: [[TMP5:%.*]] = sub i64 0, [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[TMP2]], i64 [[TMP5]] -; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x double> @llvm.vp.load.nxv2f64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.strided.load.nxv2f64.p0.i64(ptr align 8 [[TMP2]], i64 -8, <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) ; CHECK-NEXT: [[TMP7:%.*]] = fneg <vscale x 2 x double> [[VP_OP_LOAD]] ; CHECK-NEXT: [[TMP8:%.*]] = fadd <vscale x 2 x double> [[TMP7]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x double> @llvm.experimental.vp.reverse.nxv2f64(<vscale x 2 x double> [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[INDEX]] -; CHECK-NEXT: call void @llvm.vp.store.nxv2f64.p0(<vscale x 2 x double> [[TMP9]], ptr align 8 [[TMP10]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: call void @llvm.vp.store.nxv2f64.p0(<vscale x 2 x double> [[TMP8]], ptr align 8 [[TMP10]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP0]] to i64 ; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP3]], [[INDEX]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP3]] ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 @@ -129,15 +120,10 @@ ; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 2 x i8> [[VP_OP_LOAD]], zeroinitializer ; CHECK-NEXT: [[TMP3:%.*]] = sub i64 [[N]], [[INDEX]] ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[B]], i64 [[TMP3]] -; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i1> @llvm.experimental.vp.reverse.nxv2i1(<vscale x 2 x i1> [[TMP2]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) -; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[TMP0]] to i64 -; CHECK-NEXT: [[TMP7:%.*]] = sub nuw nsw i64 [[TMP6]], 1 -; CHECK-NEXT: [[TMP8:%.*]] = sub i64 0, [[TMP7]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP4]], i64 [[TMP8]] -; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP9]], <vscale x 2 x i1> [[TMP5]], i32 [[TMP0]]) -; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.reverse.nxv2i64(<vscale x 2 x i64> [[VP_OP_LOAD1]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.strided.load.nxv2i64.p0.i64(ptr align 8 [[TMP4]], i64 -8, <vscale x 2 x i1> [[TMP2]], i32 [[TMP0]]) ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]] ; CHECK-NEXT: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP10]], ptr align 8 [[TMP11]], <vscale x 2 x i1> [[TMP2]], i32 [[TMP0]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[TMP0]] to i64 ; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP6]], [[INDEX]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP6]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll index c01bd9d..5ab6573 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -68,6 +68,7 @@ ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] cse ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeDeadRecipes@3 ; CHECK-AFTER: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimize +; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::convertToReverseStridedAccess ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::narrowInterleaveGroups ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializeHeaderMask ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] printOptimizedVPlan