[LV] Support EVL for partial reduction and VPExpressionRecipe. (#205741) This patch adds the support for partial reduction with EVL tail-folding by not creating a new VPExpressionEVLRecipes but just changing the last expressionRecipes from VPReductionRecipe to VPReductionEVLRecipe. Currently, all partial reductions will be converted to VPExpressionRecipe when construction. So this patch also supports the VPExpressionRecipe for EVL.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index a72d69b..8ca56cb 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6792,14 +6792,10 @@ // Create partial reduction recipes for scaled reductions and transform // recipes to abstract recipes if it is legal and beneficial and clamp the // range for better cost estimation. - // TODO: Enable following transform when the EVL-version of extended-reduction - // and mulacc-reduction are implemented. - if (!CM.foldTailWithEVL()) { - RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx, - Range); - RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx, - Range); - } + RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx, + Range); + RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx, + Range); // Interleave memory: for each Interleave Group we marked earlier as relevant // for this VPlan, replace the Recipes widening its memory instructions with a
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index fe1342e..7d2c2fa1 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3364,7 +3364,8 @@ R.getFastMathFlagsOrNone(), cast_or_null<Instruction>(R.getUnderlyingValue()), {R.getChainOp(), R.getVecOp(), &EVL}, CondOp, - getReductionStyle(/*InLoop=*/true, R.isOrdered(), 1), + getReductionStyle(R.isInLoop(), R.isOrdered(), + R.getVFScaleFactor()), DL) {} ~VPReductionEVLRecipe() override = default; @@ -3588,6 +3589,7 @@ /// Type of the expression. ExpressionTypes ExpressionType; +public: /// Construct a new VPExpressionRecipe by internalizing recipes in \p /// ExpressionRecipes. External operands (i.e. not defined by another recipe /// in the expression) are replaced by temporary VPValues and the original @@ -3597,7 +3599,6 @@ VPExpressionRecipe(ExpressionTypes ExpressionType, ArrayRef<VPSingleDefRecipe *> ExpressionRecipes); -public: VPExpressionRecipe(VPWidenCastRecipe *Ext, VPReductionRecipe *Red) : VPExpressionRecipe(ExpressionTypes::ExtendedReduction, {Ext, Red}) {} VPExpressionRecipe(VPWidenCastRecipe *Ext, VPWidenRecipe *Neg, @@ -3670,10 +3671,13 @@ return new VPExpressionRecipe(ExpressionType, NewExpressiondRecipes); } - /// Insert the recipes of the expression back into the VPlan, directly before - /// the current recipe. Leaves the expression recipe empty, which must be - /// removed before codegen. - void decompose(); + /// Return and insert the recipes of the expression back into the VPlan, + /// directly before the current recipe. Leaves the expression recipe empty, + /// which must be removed before codegen. + SmallVector<VPSingleDefRecipe *> decompose(); + + /// Returns the expression type of this recipe. + ExpressionTypes getExpressionType() const { return ExpressionType; } unsigned getVFScaleFactor() const { auto *PR = dyn_cast<VPReductionRecipe>(ExpressionRecipes.back());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp index 42e8d87..9d8e551 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -19,6 +19,7 @@ #include "VPlanPatternMatch.h" #include "VPlanTransforms.h" #include "VPlanUtils.h" +#include "llvm/ADT/SetVector.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/IR/Intrinsics.h" @@ -240,6 +241,40 @@ return nullptr; } +// Decompose the expression recipe and transform each contained recipe into +// an EVL recipe. +static bool +optimizeExpressionRecipeToEVL(VPValue *HeaderMask, VPRecipeBase &CurRecipe, + VPValue &EVL, + SmallVector<VPRecipeBase *> &OldRecipes) { + + auto *Expr = dyn_cast<VPExpressionRecipe>(&CurRecipe); + if (!Expr) + return false; + + // Decompose first and construct with EVL recipes later. + SmallVector<VPSingleDefRecipe *> ExpressionRecipes(Expr->decompose()); + SmallSetVector<VPSingleDefRecipe *, 4> UniqueExpressionRecipes( + from_range, ExpressionRecipes); + + // Convert recipes to EVL recipes. + for (auto *R : UniqueExpressionRecipes) + if (auto *EVLR = cast_if_present<VPSingleDefRecipe>( + optimizeMaskToEVL(HeaderMask, *R, EVL))) { + EVLR->insertBefore(R); + R->replaceAllUsesWith(EVLR); + OldRecipes.push_back(R); + replace(ExpressionRecipes, R, EVLR); + } + + auto *NewExpr = + new VPExpressionRecipe(Expr->getExpressionType(), ExpressionRecipes); + ExpressionRecipes.back()->replaceAllUsesWith(NewExpr); + NewExpr->insertBefore(Expr); + OldRecipes.push_back(Expr); + return true; +} + /// Optimize away any EVL-based header masks to VP intrinsic based recipes. /// The transforms here need to preserve the original semantics. void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) { @@ -259,6 +294,9 @@ SmallVector<VPRecipeBase *> OldRecipes; for (VPUser *U : vputils::collectUsersRecursively(HeaderMask)) { VPRecipeBase *R = cast<VPRecipeBase>(U); + // Transform recipes contained by an expression recipe into EVL recipes. + if (optimizeExpressionRecipeToEVL(HeaderMask, *R, *EVL, OldRecipes)) + continue; if (auto *NewR = optimizeMaskToEVL(HeaderMask, *R, *EVL)) { NewR->insertBefore(R); for (auto [Old, New] :
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index c91ebcb..047bcb3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3427,13 +3427,15 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) { + assert(State.VF.isVector() && + "Shouldn't generate VPReductionEVLRecipe with scalar VF"); auto &Builder = State.Builder; // Propagate the fast-math flags carried by the underlying instruction. IRBuilderBase::FastMathFlagGuard FMFGuard(Builder); Builder.setFastMathFlags(getFastMathFlagsOrNone()); RecurKind Kind = getRecurrenceKind(); - Value *Prev = State.get(getChainOp(), /*IsScalar*/ true); + Value *Prev = State.get(getChainOp(), /*IsScalar*/ !isPartialReduction()); Value *VecOp = State.get(getVecOp()); Value *EVL = State.get(getEVL(), VPLane(0)); @@ -3444,7 +3446,27 @@ Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue()); Value *NewRed; - if (isOrdered()) { + if (isPartialReduction()) { + // For partial reductions, we need to generate a predicated select + // (vp.merge) since `@llvm.vector.partial.reduce()` doesn't have a vector + // predicated version. + VectorType *VecTy = cast<VectorType>(VecOp->getType()); + Value *Identity = getRecurrenceIdentity(Kind, VecTy->getElementType(), + getFastMathFlagsOrNone()); + Identity = + State.Builder.CreateVectorSplat(VecTy->getElementCount(), Identity); + + // TODO: Calculate the predicate cost for the partial reduction. + Value *NewVecOp = State.Builder.CreateIntrinsic( + VecTy, Intrinsic::vp_merge, {Mask, VecOp, Identity, EVL}); + assert((Kind == RecurKind::Add || Kind == RecurKind::FAdd) && + "Unexpected partial reduction kind"); + NewRed = State.Builder.CreateIntrinsic( + Prev->getType(), + Kind == RecurKind::Add ? Intrinsic::vector_partial_reduce_add + : Intrinsic::vector_partial_reduce_fadd, + {Prev, NewVecOp}, State.Builder.getFastMathFlags(), "partial.reduce"); + } else if (isOrdered()) { NewRed = createOrderedReduction(Builder, Kind, VecOp, Prev, Mask, EVL); } else { NewRed = createSimpleReduction(Builder, VecOp, Kind, Mask, EVL); @@ -3455,7 +3477,7 @@ (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(Kind), NewRed, Prev); } - State.set(this, NewRed, /*IsScalar*/ true); + State.set(this, NewRed, !isPartialReduction()); } InstructionCost VPReductionRecipe::computeCost(ElementCount VF, @@ -3563,7 +3585,7 @@ R->replaceUsesOfWith(LiveIn, Tmp); } -void VPExpressionRecipe::decompose() { +SmallVector<VPSingleDefRecipe *> VPExpressionRecipe::decompose() { for (auto *R : ExpressionRecipes) // Since the list could contain duplicates, make sure the recipe hasn't // already been inserted. @@ -3574,7 +3596,9 @@ LiveInPlaceholders[Idx]->replaceAllUsesWith(Op); replaceAllUsesWith(ExpressionRecipes.back()); + SmallVector<VPSingleDefRecipe *> DecomposedRecipes(ExpressionRecipes); ExpressionRecipes.clear(); + return DecomposedRecipes; } InstructionCost VPExpressionRecipe::computeCost(ElementCount VF, @@ -3682,8 +3706,23 @@ O << " = "; auto *Red = cast<VPReductionRecipe>(ExpressionRecipes.back()); unsigned Opcode = RecurrenceDescriptor::getOpcode(Red->getRecurrenceKind()); - VPValue *RdxStart = - getOperand(getNumOperands() - (Red->isConditional() ? 2 : 1)); + VPValue *Mask = getOperand(getNumOperands() - 1); + VPValue *EVL = + isa<VPReductionEVLRecipe>(Red) + ? getOperand(getNumOperands() - (Red->isConditional() ? 2 : 1)) + : nullptr; + VPValue *RdxStart = getOperand( + getNumOperands() - (Red->isConditional() ? 2 : 1) - (EVL ? 1 : 0)); + auto PrintEVLAndMask = [&]() { + if (EVL) { + O << ", "; + EVL->printAsOperand(O, SlotTracker); + } + if (Red->isConditional()) { + O << ", "; + Mask->printAsOperand(O, SlotTracker); + } + }; switch (ExpressionType) { case ExpressionTypes::NegatedExtendedReduction: @@ -3702,10 +3741,7 @@ auto *Ext0 = cast<VPWidenCastRecipe>(ExpressionRecipes[0]); O << Instruction::getOpcodeName(Ext0->getOpcode()) << " to " << *Ext0->getScalarType(); - if (Red->isConditional()) { - O << ", "; - getOperand(getNumOperands() - 1)->printAsOperand(O, SlotTracker); - } + PrintEVLAndMask(); O << ")"; break; } @@ -3726,10 +3762,7 @@ auto *Ext1 = cast<VPWidenCastRecipe>(ExpressionRecipes[1]); O << " " << Instruction::getOpcodeName(Ext1->getOpcode()) << " to " << *Ext1->getScalarType() << ")"; - if (Red->isConditional()) { - O << ", "; - getOperand(getNumOperands() - 1)->printAsOperand(O, SlotTracker); - } + PrintEVLAndMask(); O << "))"; break; } @@ -3761,10 +3794,7 @@ O << " " << Instruction::getOpcodeName(Ext1->getOpcode()) << " to " << *Ext1->getScalarType() << ")"; } - if (Red->isConditional()) { - O << ", "; - getOperand(getNumOperands() - 1)->printAsOperand(O, SlotTracker); - } + PrintEVLAndMask(); O << ")"; break; } @@ -3795,7 +3825,10 @@ void VPReductionEVLRecipe::printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { - O << Indent << "REDUCE "; + if (isPartialReduction()) + O << Indent << "PARTIAL-REDUCE "; + else + O << Indent << "REDUCE "; printAsOperand(O, SlotTracker); O << " = "; getChainOp()->printAsOperand(O, SlotTracker);
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll index 20c330e..e99c68e 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
@@ -154,25 +154,25 @@ ; TAILFOLD-NEXT: br label [[VECTOR_BODY:%.*]] ; TAILFOLD: vector.body: ; TAILFOLD-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ] -; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ] +; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ] ; TAILFOLD-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] ; TAILFOLD-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) ; TAILFOLD-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[EVL_BASED_IV]] ; TAILFOLD-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) -; TAILFOLD-NEXT: [[TMP2:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32> ; TAILFOLD-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[EVL_BASED_IV]] ; TAILFOLD-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) ; TAILFOLD-NEXT: [[TMP4:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32> +; TAILFOLD-NEXT: [[TMP2:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32> ; TAILFOLD-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP4]], [[TMP2]] -; TAILFOLD-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i32> [[TMP5]], [[VEC_PHI]] -; TAILFOLD-NEXT: [[TMP7]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP6]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]]) +; TAILFOLD-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]]) +; TAILFOLD-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP6]]) ; TAILFOLD-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64 ; TAILFOLD-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP8]], [[EVL_BASED_IV]] ; TAILFOLD-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]] ; TAILFOLD-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 ; TAILFOLD-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; TAILFOLD: middle.block: -; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP7]]) +; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]]) ; TAILFOLD-NEXT: br label [[FOR_EXIT:%.*]] ; TAILFOLD: for.exit: ; TAILFOLD-NEXT: ret i32 [[TMP10]] @@ -345,25 +345,25 @@ ; TAILFOLD-NEXT: br label [[VECTOR_BODY:%.*]] ; TAILFOLD: vector.body: ; TAILFOLD-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ] -; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ] +; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ] ; TAILFOLD-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] ; TAILFOLD-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) ; TAILFOLD-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[EVL_BASED_IV]] ; TAILFOLD-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) -; TAILFOLD-NEXT: [[TMP2:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32> ; TAILFOLD-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[EVL_BASED_IV]] ; TAILFOLD-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) ; TAILFOLD-NEXT: [[TMP4:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32> +; TAILFOLD-NEXT: [[TMP2:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32> ; TAILFOLD-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP4]], [[TMP2]] -; TAILFOLD-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i32> [[TMP5]], [[VEC_PHI]] -; TAILFOLD-NEXT: [[TMP7]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP6]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]]) +; TAILFOLD-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]]) +; TAILFOLD-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP6]]) ; TAILFOLD-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64 ; TAILFOLD-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP8]], [[EVL_BASED_IV]] ; TAILFOLD-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]] ; TAILFOLD-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 ; TAILFOLD-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; TAILFOLD: middle.block: -; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP7]]) +; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]]) ; TAILFOLD-NEXT: br label [[FOR_EXIT:%.*]] ; TAILFOLD: for.exit: ; TAILFOLD-NEXT: ret i32 [[TMP10]] @@ -536,25 +536,25 @@ ; TAILFOLD-NEXT: br label [[VECTOR_BODY:%.*]] ; TAILFOLD: vector.body: ; TAILFOLD-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ] -; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ] +; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ] ; TAILFOLD-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] ; TAILFOLD-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) ; TAILFOLD-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[EVL_BASED_IV]] ; TAILFOLD-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) -; TAILFOLD-NEXT: [[TMP2:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32> ; TAILFOLD-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[EVL_BASED_IV]] ; TAILFOLD-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) ; TAILFOLD-NEXT: [[TMP4:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32> +; TAILFOLD-NEXT: [[TMP2:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32> ; TAILFOLD-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP4]], [[TMP2]] -; TAILFOLD-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i32> [[TMP5]], [[VEC_PHI]] -; TAILFOLD-NEXT: [[TMP7]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP6]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]]) +; TAILFOLD-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]]) +; TAILFOLD-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP6]]) ; TAILFOLD-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64 ; TAILFOLD-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP8]], [[EVL_BASED_IV]] ; TAILFOLD-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]] ; TAILFOLD-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 ; TAILFOLD-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; TAILFOLD: middle.block: -; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP7]]) +; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]]) ; TAILFOLD-NEXT: br label [[FOR_EXIT:%.*]] ; TAILFOLD: for.exit: ; TAILFOLD-NEXT: ret i32 [[TMP10]] @@ -726,25 +726,25 @@ ; TAILFOLD-NEXT: br label [[VECTOR_BODY:%.*]] ; TAILFOLD: vector.body: ; TAILFOLD-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ] -; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ] +; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ] ; TAILFOLD-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] ; TAILFOLD-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) ; TAILFOLD-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[EVL_BASED_IV]] ; TAILFOLD-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) -; TAILFOLD-NEXT: [[TMP2:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32> ; TAILFOLD-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[EVL_BASED_IV]] ; TAILFOLD-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) ; TAILFOLD-NEXT: [[TMP4:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32> +; TAILFOLD-NEXT: [[TMP2:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32> ; TAILFOLD-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP4]], [[TMP2]] -; TAILFOLD-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i32> [[TMP5]], [[VEC_PHI]] -; TAILFOLD-NEXT: [[TMP7]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP6]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]]) +; TAILFOLD-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]]) +; TAILFOLD-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP6]]) ; TAILFOLD-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64 ; TAILFOLD-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP8]], [[EVL_BASED_IV]] ; TAILFOLD-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]] ; TAILFOLD-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 ; TAILFOLD-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; TAILFOLD: middle.block: -; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP7]]) +; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]]) ; TAILFOLD-NEXT: br label [[FOR_EXIT:%.*]] ; TAILFOLD: for.exit: ; TAILFOLD-NEXT: ret i32 [[TMP10]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll new file mode 100644 index 0000000..2b3cf2c --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll
@@ -0,0 +1,858 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6 +; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+experimental-zvdot4a8i -S < %s | FileCheck %s --check-prefixes=CHECK +; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+experimental-zvdot4a8i -scalable-vectorization=off -tail-folding-policy=dont-fold-tail -S < %s | FileCheck %s --check-prefixes=CHECK-FIXED + +define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 { +; CHECK-LABEL: define i32 @pred_reduction( +; CHECK-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[PARTIAL_REDUCE4:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]]) +; CHECK-NEXT: [[PARTIAL_REDUCE:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32> +; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[BIN_RDX]], <vscale x 4 x i32> [[PARTIAL_REDUCE4]] +; CHECK-NEXT: [[TMP6]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32> [[PARTIAL_REDUCE4]], i32 [[TMP0]]) +; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP7]], [[INDEX]] +; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]] +; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP6]]) +; CHECK-NEXT: br label %[[SCALAR_PH:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: ret i32 [[TMP9]] +; +; CHECK-FIXED-LABEL: define i32 @pred_reduction( +; CHECK-FIXED-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-FIXED-NEXT: [[ENTRY:.*:]] +; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16 +; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK-FIXED: [[VECTOR_PH]]: +; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15 +; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK-FIXED: [[VECTOR_BODY]]: +; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI4:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1 +; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1 +; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer +; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer +; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[TMP10:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP7:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP8:%.*]] = add <8 x i32> [[VEC_PHI]], [[TMP10]] +; CHECK-FIXED-NEXT: [[TMP12:%.*]] = add <8 x i32> [[VEC_PHI1]], [[TMP7]] +; CHECK-FIXED-NEXT: [[PREDPHI]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP8]], <8 x i32> [[VEC_PHI]] +; CHECK-FIXED-NEXT: [[PREDPHI4]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP12]], <8 x i32> [[VEC_PHI1]] +; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 +; CHECK-FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-FIXED-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-FIXED: [[MIDDLE_BLOCK]]: +; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[PREDPHI4]], [[PREDPHI]] +; CHECK-FIXED-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]]) +; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK-FIXED: [[SCALAR_PH]]: +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ] + %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv + %c = load i8, ptr %arrayidx, align 1 + %tobool.not = icmp eq i8 %c, 0 + br i1 %tobool.not, label %for.inc, label %if.then + +if.then: + %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv + %val = load i8, ptr %arrayidx2, align 1 + %conv = zext i8 %val to i32 + %add = add nsw i32 %sum, %conv + br label %for.inc + +for.inc: + %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret i32 %sum.1 +} + +define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 { +; CHECK-LABEL: define i32 @pred_reduction_sext( +; CHECK-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[PARTIAL_REDUCE4:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]]) +; CHECK-NEXT: [[PARTIAL_REDUCE:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32> +; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[BIN_RDX]], <vscale x 4 x i32> [[PARTIAL_REDUCE4]] +; CHECK-NEXT: [[TMP6]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32> [[PARTIAL_REDUCE4]], i32 [[TMP0]]) +; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP7]], [[INDEX]] +; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]] +; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP6]]) +; CHECK-NEXT: br label %[[SCALAR_PH:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: ret i32 [[TMP9]] +; +; CHECK-FIXED-LABEL: define i32 @pred_reduction_sext( +; CHECK-FIXED-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-FIXED-NEXT: [[ENTRY:.*:]] +; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16 +; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK-FIXED: [[VECTOR_PH]]: +; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15 +; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK-FIXED: [[VECTOR_BODY]]: +; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI4:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1 +; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1 +; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer +; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer +; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[TMP10:%.*]] = sext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP7:%.*]] = sext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP8:%.*]] = add <8 x i32> [[VEC_PHI]], [[TMP10]] +; CHECK-FIXED-NEXT: [[TMP12:%.*]] = add <8 x i32> [[VEC_PHI1]], [[TMP7]] +; CHECK-FIXED-NEXT: [[PREDPHI]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP8]], <8 x i32> [[VEC_PHI]] +; CHECK-FIXED-NEXT: [[PREDPHI4]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP12]], <8 x i32> [[VEC_PHI1]] +; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 +; CHECK-FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-FIXED-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK-FIXED: [[MIDDLE_BLOCK]]: +; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[PREDPHI4]], [[PREDPHI]] +; CHECK-FIXED-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]]) +; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK-FIXED: [[SCALAR_PH]]: +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ] + %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv + %c = load i8, ptr %arrayidx, align 1 + %tobool.not = icmp eq i8 %c, 0 + br i1 %tobool.not, label %for.inc, label %if.then + +if.then: + %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv + %val = load i8, ptr %arrayidx2, align 1 + %conv = sext i8 %val to i32 + %add = add nsw i32 %sum, %conv + br label %for.inc + +for.inc: + %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret i32 %sum.1 +} + +define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 { +; CHECK-LABEL: define i32 @pred_reduction_dotprod( +; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]]) +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP10]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]]) +; CHECK-NEXT: [[TMP5:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD2]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = mul nuw nsw <vscale x 4 x i32> [[TMP5]], [[TMP6]] +; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]]) +; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP12]]) +; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP9]], [[INDEX]] +; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] +; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]]) +; CHECK-NEXT: br label %[[SCALAR_PH:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: ret i32 [[TMP11]] +; +; CHECK-FIXED-LABEL: define i32 @pred_reduction_dotprod( +; CHECK-FIXED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-FIXED-NEXT: [[ENTRY:.*:]] +; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16 +; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK-FIXED: [[VECTOR_PH]]: +; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15 +; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK-FIXED: [[VECTOR_BODY]]: +; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1 +; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1 +; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer +; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer +; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP7]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP7]], <8 x i1> [[TMP2]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP16]], <8 x i1> [[TMP3]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[TMP8:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP9:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD4]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP10:%.*]] = mul nuw nsw <8 x i32> [[TMP8]], [[TMP9]] +; CHECK-FIXED-NEXT: [[TMP11:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP10]], <8 x i32> zeroinitializer +; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI]], <8 x i32> [[TMP11]]) +; CHECK-FIXED-NEXT: [[TMP18:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP13:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD5]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP14:%.*]] = mul nuw nsw <8 x i32> [[TMP18]], [[TMP13]] +; CHECK-FIXED-NEXT: [[TMP15:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP14]], <8 x i32> zeroinitializer +; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE6]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI1]], <8 x i32> [[TMP15]]) +; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 +; CHECK-FIXED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-FIXED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK-FIXED: [[MIDDLE_BLOCK]]: +; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <2 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]] +; CHECK-FIXED-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[BIN_RDX]]) +; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK-FIXED: [[SCALAR_PH]]: +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ] + %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv + %c = load i8, ptr %arrayidx, align 1 + %tobool.not = icmp eq i8 %c, 0 + br i1 %tobool.not, label %for.inc, label %if.then + +if.then: + %arrayidx2 = getelementptr inbounds nuw i8, ptr %a, i64 %iv + %load.a = load i8, ptr %arrayidx2, align 1 + %arrayidx4 = getelementptr inbounds nuw i8, ptr %b, i64 %iv + %load.b = load i8, ptr %arrayidx4, align 1 + %ext.a = zext i8 %load.a to i32 + %ext.b = zext i8 %load.b to i32 + %mul = mul nuw nsw i32 %ext.a, %ext.b + %add = add nsw i32 %sum, %mul + br label %for.inc + +for.inc: + %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret i32 %sum.1 +} + +define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 { +; CHECK-LABEL: define i32 @pred_sub_reduction( +; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]]) +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP10]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]]) +; CHECK-NEXT: [[TMP5:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD2]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = mul nuw nsw <vscale x 4 x i32> [[TMP5]], [[TMP6]] +; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]]) +; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP11]]) +; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP9]], [[INDEX]] +; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] +; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]]) +; CHECK-NEXT: [[TMP22:%.*]] = sub i32 0, [[TMP21]] +; CHECK-NEXT: br label %[[SCALAR_PH:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: ret i32 [[TMP22]] +; +; CHECK-FIXED-LABEL: define i32 @pred_sub_reduction( +; CHECK-FIXED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-FIXED-NEXT: [[ENTRY:.*:]] +; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16 +; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK-FIXED: [[VECTOR_PH]]: +; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15 +; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK-FIXED: [[VECTOR_BODY]]: +; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1 +; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1 +; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer +; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer +; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP7]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP7]], <8 x i1> [[TMP2]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP16]], <8 x i1> [[TMP3]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[TMP8:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP9:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD4]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP10:%.*]] = mul nuw nsw <8 x i32> [[TMP8]], [[TMP9]] +; CHECK-FIXED-NEXT: [[TMP11:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP10]], <8 x i32> zeroinitializer +; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI]], <8 x i32> [[TMP11]]) +; CHECK-FIXED-NEXT: [[TMP17:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP18:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD5]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP19:%.*]] = mul nuw nsw <8 x i32> [[TMP17]], [[TMP18]] +; CHECK-FIXED-NEXT: [[TMP15:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP19]], <8 x i32> zeroinitializer +; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE6]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI1]], <8 x i32> [[TMP15]]) +; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 +; CHECK-FIXED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-FIXED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK-FIXED: [[MIDDLE_BLOCK]]: +; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <2 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]] +; CHECK-FIXED-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[BIN_RDX]]) +; CHECK-FIXED-NEXT: [[TMP14:%.*]] = sub i32 0, [[TMP13]] +; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK-FIXED: [[SCALAR_PH]]: +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ] + %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv + %c = load i8, ptr %arrayidx, align 1 + %tobool.not = icmp eq i8 %c, 0 + br i1 %tobool.not, label %for.inc, label %if.then + +if.then: + %arrayidx2 = getelementptr inbounds nuw i8, ptr %a, i64 %iv + %load.a = load i8, ptr %arrayidx2, align 1 + %arrayidx4 = getelementptr inbounds nuw i8, ptr %b, i64 %iv + %load.b = load i8, ptr %arrayidx4, align 1 + %ext.a = zext i8 %load.a to i32 + %ext.b = zext i8 %load.b to i32 + %mul = mul nuw nsw i32 %ext.a, %ext.b + %sub = sub nsw i32 %sum, %mul + br label %for.inc + +for.inc: + %sum.1 = phi i32 [ %sub, %if.then ], [ %sum, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret i32 %sum.1 +} + +define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64 %N) #0 { +; CHECK-LABEL: define i32 @chained_pred_reduction( +; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]]) +; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP6]] +; CHECK-NEXT: [[PARTIAL_REDUCE8:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[VEC_PHI]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[PARTIAL_REDUCE7:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD2]] to <vscale x 4 x i32> +; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]] +; CHECK-NEXT: [[TMP9]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[BIN_RDX]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]]) +; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP10]], [[INDEX]] +; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]] +; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP9]]) +; CHECK-NEXT: br label %[[SCALAR_PH:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: ret i32 [[TMP12]] +; +; CHECK-FIXED-LABEL: define i32 @chained_pred_reduction( +; CHECK-FIXED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-FIXED-NEXT: [[ENTRY:.*:]] +; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16 +; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK-FIXED: [[VECTOR_PH]]: +; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15 +; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK-FIXED: [[VECTOR_BODY]]: +; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1 +; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1 +; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer +; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer +; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[TMP10:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP7:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP8:%.*]] = add <8 x i32> [[VEC_PHI]], [[TMP10]] +; CHECK-FIXED-NEXT: [[TMP16:%.*]] = add <8 x i32> [[VEC_PHI1]], [[TMP7]] +; CHECK-FIXED-NEXT: [[PREDPHI:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP8]], <8 x i32> [[VEC_PHI]] +; CHECK-FIXED-NEXT: [[PREDPHI4:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP16]], <8 x i32> [[VEC_PHI1]] +; CHECK-FIXED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x i8>, ptr [[TMP9]], align 1 +; CHECK-FIXED-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i8>, ptr [[TMP11]], align 1 +; CHECK-FIXED-NEXT: [[TMP18:%.*]] = zext <8 x i8> [[WIDE_LOAD5]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP13:%.*]] = zext <8 x i8> [[WIDE_LOAD6]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP14]] = add <8 x i32> [[PREDPHI]], [[TMP18]] +; CHECK-FIXED-NEXT: [[TMP15]] = add <8 x i32> [[PREDPHI4]], [[TMP13]] +; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 +; CHECK-FIXED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-FIXED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; CHECK-FIXED: [[MIDDLE_BLOCK]]: +; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[TMP15]], [[TMP14]] +; CHECK-FIXED-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]]) +; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK-FIXED: [[SCALAR_PH]]: +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.inc ] + %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv + %c = load i8, ptr %arrayidx, align 1 + %tobool.not = icmp eq i8 %c, 0 + br i1 %tobool.not, label %for.inc, label %if.then + +if.then: + %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv + %val = load i8, ptr %arrayidx2, align 1 + %conv = zext i8 %val to i32 + %add = add nsw i32 %sum, %conv + br label %for.inc + +for.inc: + %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ] + %b.gep = getelementptr inbounds nuw i8, ptr %src_b, i64 %iv + %bval = load i8, ptr %b.gep, align 1 + %bconv = zext i8 %bval to i32 + %sum.2 = add nsw i32 %sum.1, %bconv + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret i32 %sum.2 +} + +define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %N) #0 { +; CHECK-LABEL: define i32 @reduction_before_pred( +; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP2:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32> +; CHECK-NEXT: [[PARTIAL_REDUCE8:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP2]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD1]], zeroinitializer +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP12]], <vscale x 4 x i1> [[TMP5]], i32 [[TMP0]]) +; CHECK-NEXT: [[PARTIAL_REDUCE7:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD2]] to <vscale x 4 x i32> +; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i32> [[BIN_RDX]], <vscale x 4 x i32> [[PARTIAL_REDUCE8]] +; CHECK-NEXT: [[TMP9]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]]) +; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP10]], [[INDEX]] +; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]] +; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP9]]) +; CHECK-NEXT: br label %[[SCALAR_PH:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: ret i32 [[TMP13]] +; +; CHECK-FIXED-LABEL: define i32 @reduction_before_pred( +; CHECK-FIXED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-FIXED-NEXT: [[ENTRY:.*:]] +; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16 +; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK-FIXED: [[VECTOR_PH]]: +; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15 +; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK-FIXED: [[VECTOR_BODY]]: +; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI6:%.*]], %[[VECTOR_BODY]] ] +; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1 +; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1 +; CHECK-FIXED-NEXT: [[TMP2:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP3:%.*]] = zext <8 x i8> [[WIDE_LOAD2]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP6:%.*]] = add <8 x i32> [[VEC_PHI]], [[TMP2]] +; CHECK-FIXED-NEXT: [[TMP5:%.*]] = add <8 x i32> [[VEC_PHI1]], [[TMP3]] +; CHECK-FIXED-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_LOAD3:%.*]] = load <8 x i8>, ptr [[TMP7]], align 1 +; CHECK-FIXED-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP10]], align 1 +; CHECK-FIXED-NEXT: [[TMP8:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD3]], zeroinitializer +; CHECK-FIXED-NEXT: [[TMP16:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD4]], zeroinitializer +; CHECK-FIXED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]] +; CHECK-FIXED-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP9]], i64 8 +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP9]], <8 x i1> [[TMP8]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP11]], <8 x i1> [[TMP16]], <8 x i8> poison) +; CHECK-FIXED-NEXT: [[TMP18:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP13:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD5]] to <8 x i32> +; CHECK-FIXED-NEXT: [[TMP14:%.*]] = add <8 x i32> [[TMP6]], [[TMP18]] +; CHECK-FIXED-NEXT: [[TMP15:%.*]] = add <8 x i32> [[TMP5]], [[TMP13]] +; CHECK-FIXED-NEXT: [[PREDPHI]] = select <8 x i1> [[TMP8]], <8 x i32> [[TMP14]], <8 x i32> [[TMP6]] +; CHECK-FIXED-NEXT: [[PREDPHI6]] = select <8 x i1> [[TMP16]], <8 x i32> [[TMP15]], <8 x i32> [[TMP5]] +; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 +; CHECK-FIXED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-FIXED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] +; CHECK-FIXED: [[MIDDLE_BLOCK]]: +; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[PREDPHI6]], [[PREDPHI]] +; CHECK-FIXED-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]]) +; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK-FIXED: [[SCALAR_PH]]: +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.inc ] + + %b.gep = getelementptr inbounds nuw i8, ptr %src_b, i64 %iv + %bval = load i8, ptr %b.gep, align 1 + %bconv = zext i8 %bval to i32 + %sum.1 = add nsw i32 %sum, %bconv + + %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv + %c = load i8, ptr %arrayidx, align 1 + %tobool.not = icmp eq i8 %c, 0 + br i1 %tobool.not, label %for.inc, label %if.then + +if.then: + %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv + %val = load i8, ptr %arrayidx2, align 1 + %conv = zext i8 %val to i32 + %add = add nsw i32 %sum.1, %conv + br label %for.inc + +for.inc: + %sum.2 = phi i32 [ %sum.1, %for.body ], [ %add, %if.then ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret i32 %sum.2 +} + +; UTC_ARGS: --disable + +; Negative test - expect not to generate a partial reduction as phi +; has two update values +define i32 @partial_reduce_if_else(ptr %a, ptr %b, ptr %cond, i64 %N) #0 { +; CHECK-LABEL: define i32 @partial_reduce_if_else( +; CHECK-NOT: llvm.vector.partial.reduce +; CHECK: call i32 @llvm.vector.reduce.add +; +; CHECK-FIXED-LABEL: define i32 @partial_reduce_if_else( +; CHECK-FIXED-NOT: llvm.vector.partial.reduce +; CHECK-FIXED: call i32 @llvm.vector.reduce.add +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %sum = phi i32 [ 0, %entry ], [ %sum.next, %for.inc ] + + %arrayidx.cond = getelementptr inbounds nuw i8, ptr %cond, i64 %iv + %c = load i8, ptr %arrayidx.cond, align 1 + %tobool.not = icmp eq i8 %c, 0 + br i1 %tobool.not, label %if.else, label %if.then + +if.then: + %arrayidx.a = getelementptr inbounds nuw i8, ptr %a, i64 %iv + %aval = load i8, ptr %arrayidx.a, align 1 + %a.ext = zext i8 %aval to i32 + %add.a = add nsw i32 %sum, %a.ext + br label %for.inc + +if.else: + %arrayidx.b = getelementptr inbounds nuw i8, ptr %b, i64 %iv + %bval = load i8, ptr %arrayidx.b, align 1 + %b.ext = zext i8 %bval to i32 + %add.b = add nsw i32 %sum, %b.ext + br label %for.inc + +for.inc: + %sum.next = phi i32 [ %add.a, %if.then ], [ %add.b, %if.else ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret i32 %sum.next +} + +; Negative test - expect not to generate the partial reduction as +; there are more than two incomng values to the predicated phi. +define i32 @partial_reduce_three_incoming(ptr %a, ptr %b, ptr %cond_a, ptr %cond_b, i64 %N) #0 { +; CHECK-LABEL: define i32 @partial_reduce_three_incoming( +; CHECK-NOT: llvm.vector.partial.reduce +; CHECK: call i32 @llvm.vector.reduce.add +; +; CHECK-FIXED-LABEL: define i32 @partial_reduce_three_incoming( +; CHECK-FIXED-NOT: llvm.vector.partial.reduce +; CHECK-FIXED: call i32 @llvm.vector.reduce.add +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %sum = phi i32 [ 0, %entry ], [ %sum.next, %for.inc ] + + %arrayidx.cond.a = getelementptr inbounds nuw i8, ptr %cond_a, i64 %iv + %ca = load i8, ptr %arrayidx.cond.a, align 1 + %cond.a = icmp ne i8 %ca, 0 + br i1 %cond.a, label %if.then, label %if.else + +if.then: + %arrayidx.a = getelementptr inbounds nuw i8, ptr %a, i64 %iv + %aval = load i8, ptr %arrayidx.a, align 1 + %a.ext = zext i8 %aval to i32 + %add.a = add nsw i32 %sum, %a.ext + br label %for.inc + +if.else: + %arrayidx.cond.b = getelementptr inbounds nuw i8, ptr %cond_b, i64 %iv + %cb = load i8, ptr %arrayidx.cond.b, align 1 + %cond.b = icmp ne i8 %cb, 0 + br i1 %cond.b, label %if.else.then, label %if.else.end + +if.else.then: + %arrayidx.b = getelementptr inbounds nuw i8, ptr %b, i64 %iv + %bval = load i8, ptr %arrayidx.b, align 1 + %b.ext = zext i8 %bval to i32 + %add.b = add nsw i32 %sum, %b.ext + br label %for.inc + +if.else.end: + br label %for.inc + +for.inc: + %sum.next = phi i32 [ %add.a, %if.then ], + [ %add.b, %if.else.then ], + [ %sum, %if.else.end ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret i32 %sum.next +} + +; Negative test - chained reduction with mixed scale factors +define i32 @partial_reduce_mixed_scale_factor(ptr %a8, ptr %b16, + ptr %cond, i64 %N) #0 { +; CHECK-INTERLEAVE1-LABEL: define i32 @partial_reduce_mixed_scale_factor( +; CHECK-INTERLEAVE1-NOT: llvm.vector.partial.reduce +; CHECK-INTERLEAVE1: call i32 @llvm.vector.reduce.add +; +; CHECK-FIXED-LABEL: define i32 @partial_reduce_mixed_scale_factor( +; CHECK-FIXED-NOT: llvm.vector.partial.reduce +; CHECK-FIXED: call i32 @llvm.vector.reduce.add +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.inc ] + + %a.gep = getelementptr inbounds nuw i8, ptr %a8, i64 %iv + %aval = load i8, ptr %a.gep, align 1 + %a.ext = zext i8 %aval to i32 + %sum.1 = add nsw i32 %sum, %a.ext + + %cond.gep = getelementptr inbounds nuw i8, ptr %cond, i64 %iv + %c = load i8, ptr %cond.gep, align 1 + %tobool.not = icmp eq i8 %c, 0 + br i1 %tobool.not, label %for.inc, label %if.then + +if.then: + %b.gep = getelementptr inbounds nuw i16, ptr %b16, i64 %iv + %bval = load i16, ptr %b.gep, align 2 + %b.ext = zext i16 %bval to i32 + %add = add nsw i32 %sum.1, %b.ext + br label %for.inc + +for.inc: + %sum.2 = phi i32 [ %add, %if.then ], [ %sum.1, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret i32 %sum.2 +} + +; Negative test - the incoming value doesn't match the accumulator +; the passthrough value is %sum, not the previous chain value %sum.1. +define i32 @partial_reduce_bad_passthrough(ptr %src, ptr noalias %src_b, + ptr %cond, i64 %N) #0 { +; CHECK-LABEL: define i32 @partial_reduce_bad_passthrough( +; CHECK-NOT: llvm.vector.partial.reduce +; CHECK: call i32 @llvm.vector.reduce.add +; +; CHECK-FIXED-LABEL: define i32 @partial_reduce_bad_passthrough( +; CHECK-FIXED-NOT: llvm.vector.partial.reduce +; CHECK-FIXED: call i32 @llvm.vector.reduce.add +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] + %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.inc ] + + %b.gep = getelementptr inbounds nuw i8, ptr %src_b, i64 %iv + %bval = load i8, ptr %b.gep, align 1 + %bconv = zext i8 %bval to i32 + %sum.1 = add nsw i32 %sum, %bconv + + %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv + %c = load i8, ptr %arrayidx, align 1 + %tobool.not = icmp eq i8 %c, 0 + br i1 %tobool.not, label %for.inc, label %if.then + +if.then: + %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv + %val = load i8, ptr %arrayidx2, align 1 + %conv = zext i8 %val to i32 + %add = add nsw i32 %sum.1, %conv + br label %for.inc + +for.inc: + %sum.2 = phi i32 [ %add, %if.then ], [ %sum, %for.body ] + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret i32 %sum.2 +} +
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce.ll new file mode 100644 index 0000000..73b204a --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce.ll
@@ -0,0 +1,197 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6 +; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+experimental-zvdot4a8i -S %s | FileCheck %s + +define i32 @partial_reduce_dotprod(ptr %a, ptr %b, ptr %c, ptr %d, i64 %N) #0 { +; CHECK-LABEL: define i32 @partial_reduce_dotprod( +; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP7:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP4:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP7]], [[TMP4]] +; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]]) +; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP6]]) +; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP8]], [[INDEX]] +; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]] +; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]]) +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret i32 [[TMP10]] +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %accum = phi i32 [ 0, %entry ], [ %add.1, %loop ] + %gep.a = getelementptr inbounds nuw i8, ptr %a, i64 %iv + %load.a = load i8, ptr %gep.a, align 1 + %ext.a = sext i8 %load.a to i32 + %gep.b = getelementptr inbounds nuw i8, ptr %b, i64 %iv + %load.b = load i8, ptr %gep.b, align 1 + %ext.b = sext i8 %load.b to i32 + %mul.1 = mul i32 %ext.a, %ext.b + %add.1 = add i32 %mul.1, %accum + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret i32 %add.1 +} + +define i32 @chained_partial_reduce_dotprod(ptr %a, ptr %b, ptr %c, ptr %d, i64 %N) #0 { +; CHECK-LABEL: define i32 @chained_partial_reduce_dotprod( +; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP4:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP9]], [[TMP4]] +; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]]) +; CHECK-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP6]]) +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[D]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD3:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]]) +; CHECK-NEXT: [[TMP13:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD2]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP10:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD3]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP11:%.*]] = mul <vscale x 4 x i32> [[TMP13]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]]) +; CHECK-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]], <vscale x 4 x i32> [[TMP12]]) +; CHECK-NEXT: [[TMP14:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP14]], [[INDEX]] +; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]] +; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE4]]) +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret i32 [[TMP15]] +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %accum = phi i32 [ 0, %entry ], [ %add.2, %loop ] + %gep.a = getelementptr inbounds nuw i8, ptr %a, i64 %iv + %load.a = load i8, ptr %gep.a, align 1 + %ext.a = sext i8 %load.a to i32 + %gep.b = getelementptr inbounds nuw i8, ptr %b, i64 %iv + %load.b = load i8, ptr %gep.b, align 1 + %ext.b = sext i8 %load.b to i32 + %mul.1 = mul i32 %ext.a, %ext.b + %add.1 = add i32 %mul.1, %accum + %gep.c = getelementptr inbounds nuw i8, ptr %c, i64 %iv + %load.c = load i8, ptr %gep.c, align 1 + %ext.c = sext i8 %load.c to i32 + %gep.d = getelementptr inbounds nuw i8, ptr %d, i64 %iv + %load.d = load i8, ptr %gep.d, align 1 + %ext.d = sext i8 %load.d to i32 + %mul.2 = mul i32 %ext.c, %ext.d + %add.2 = add i32 %mul.2, %add.1 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret i32 %add.2 +} + +; Same as chained_partial_reduce_dotprod, but the first link multiplies +; IV-derived values instead of masked loads. +define i32 @chained_reduce_iv_derived_first_link(ptr %c, ptr %d, i64 %N) #0 { +; CHECK-LABEL: define i32 @chained_reduce_iv_derived_first_link( +; CHECK-SAME: ptr [[C:%.*]], ptr [[D:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.stepvector.nxv4i8() +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i8> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 +; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i8 +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[TMP3]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = sext <vscale x 4 x i8> [[VEC_IND]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP4]], [[TMP4]] +; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP1]]) +; CHECK-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP9]]) +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]]) +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[D]], i64 [[INDEX]] +; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]]) +; CHECK-NEXT: [[TMP8:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP10:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32> +; CHECK-NEXT: [[TMP11:%.*]] = mul <vscale x 4 x i32> [[TMP8]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP1]]) +; CHECK-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]], <vscale x 4 x i32> [[TMP12]]) +; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP2]], [[INDEX]] +; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP2]] +; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i8> [[VEC_IND]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE2]]) +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret i32 [[TMP14]] +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.body ] + %trunc.iv = trunc i64 %iv to i8 + %ext.iv = sext i8 %trunc.iv to i32 + %mul.1 = mul i32 %ext.iv, %ext.iv + %sum.1 = add i32 %mul.1, %sum + %gep.c = getelementptr inbounds nuw i8, ptr %c, i64 %iv + %load.c = load i8, ptr %gep.c, align 1 + %ext.c = sext i8 %load.c to i32 + %gep.d = getelementptr inbounds nuw i8, ptr %d, i64 %iv + %load.d = load i8, ptr %gep.d, align 1 + %ext.d = sext i8 %load.d to i32 + %mul.2 = mul i32 %ext.c, %ext.d + %sum.2 = add i32 %mul.2, %sum.1 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %N + br i1 %exitcond.not, label %exit, label %for.body + +exit: + ret i32 %sum.2 +}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll new file mode 100644 index 0000000..e971ff7 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
@@ -0,0 +1,68 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "ir-bb<for.exit>" --version 6 +; RUN: opt -mtriple riscv64 -passes=loop-vectorize -force-vector-width=4 -scalable-vectorization=on -mattr=+v,+experimental-zvdot4a8i -disable-output 2>&1 -vplan-print-after=optimizeEVLMasks < %s | FileCheck %s + +define i32 @vdota4(ptr %a, ptr %b) #0 { +; CHECK-LABEL: VPlan for loop in 'vdota4' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<1024> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4> +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[VP2]]>, vp<[[VP10:%[0-9]+]]> (VF scaled by 1/4) +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<1024>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]> +; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[VP7]]> +; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i8, ir<%gep.a>, ir<1> +; CHECK-NEXT: WIDEN ir<%load.a> = vp.load vp<[[VP8]]>, vp<%evl> +; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[VP7]]> +; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, ir<%gep.b>, ir<1> +; CHECK-NEXT: WIDEN ir<%load.b> = vp.load vp<[[VP9]]>, vp<%evl> +; CHECK-NEXT: EXPRESSION vp<[[VP10]]> = ir<%accum> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> sext to i32), vp<%evl>) +; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add nuw vp<[[VP11]]>, vp<[[VP5]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add) vp<[[VP10]]> +; CHECK-NEXT: Successor(s): ir-bb<for.exit> +; +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %accum = phi i32 [ 0, %entry ], [ %add, %for.body ] + %gep.a = getelementptr i8, ptr %a, i64 %iv + %load.a = load i8, ptr %gep.a, align 1 + %ext.a = sext i8 %load.a to i32 + %gep.b = getelementptr i8, ptr %b, i64 %iv + %load.b = load i8, ptr %gep.b, align 1 + %ext.b = zext i8 %load.b to i32 + %mul = mul i32 %ext.b, %ext.a + %add = add i32 %mul, %accum + %iv.next = add i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, 1024 + br i1 %exitcond.not, label %for.exit, label %for.body + +for.exit: + ret i32 %add +}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll new file mode 100644 index 0000000..2daf749 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
@@ -0,0 +1,950 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "ir-bb<exit>" --version 6 +; RUN: opt -passes=loop-vectorize -vplan-print-after="optimizeEVLMasks" -mtriple riscv64 -mattr=+v,+experimental-zvdot4a8i -force-vector-width=4 -scalable-vectorization=on -prefer-inloop-reductions -disable-output %s 2>&1 | FileCheck %s + +; Tests for printing VPlans with reductions. + +define float @print_reduction(i64 %n, ptr noalias %y) { +; CHECK-LABEL: VPlan for loop in 'print_reduction' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<%n> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector fast ir<0.000000e+00>, ir<0.000000e+00>, ir<1> +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (fadd) fast vp<[[VP2]]>, ir<%red.next> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP7]]> +; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds float, ir<%arrayidx>, ir<1> +; CHECK-NEXT: WIDEN ir<%lv> = vp.load vp<[[VP8]]>, vp<%evl> +; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast vp.reduce.fadd (ir<%lv>, vp<%evl>) +; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP9]]>, vp<[[VP5]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP9]]> +; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (fadd, in-loop) fast ir<%red.next> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: ; preds = %entry, %loop + %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] + %red = phi float [ %red.next, %loop ], [ 0.0, %entry ] + %arrayidx = getelementptr inbounds float, ptr %y, i64 %iv + %lv = load float, ptr %arrayidx, align 4 + %red.next = fadd fast float %lv, %red + %iv.next = add i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, %n + br i1 %exitcond, label %exit, label %loop + +exit: ; preds = %loop, %entry + ret float %red.next +} + +define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr noalias %dst) { +; CHECK-LABEL: VPlan for loop in 'print_reduction_with_invariant_store' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<%n> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector fast ir<0.000000e+00>, ir<0.000000e+00>, ir<1> +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (fadd) fast vp<[[VP2]]>, ir<%red.next> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP7]]> +; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds float, ir<%arrayidx>, ir<1> +; CHECK-NEXT: WIDEN ir<%lv> = vp.load vp<[[VP8]]>, vp<%evl> +; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast vp.reduce.fadd (ir<%lv>, vp<%evl>) +; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP9]]>, vp<[[VP5]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP9]]> +; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (fadd, in-loop) fast ir<%red.next> +; CHECK-NEXT: CLONE store vp<[[VP11]]>, ir<%dst> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: ; preds = %entry, %loop + %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] + %red = phi float [ %red.next, %loop ], [ 0.0, %entry ] + %arrayidx = getelementptr inbounds float, ptr %y, i64 %iv + %lv = load float, ptr %arrayidx, align 4 + %red.next = fadd fast float %lv, %red + store float %red.next, ptr %dst, align 4 + %iv.next = add i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, %n + br i1 %exitcond, label %exit, label %loop + +exit: ; preds = %loop, %entry + ret void +} + +define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { +; CHECK-LABEL: VPlan for loop in 'print_fmuladd_strict' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<%n> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector reassoc nnan ninf nsz ir<0.000000e+00>, ir<0.000000e+00>, ir<1> +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi (fmuladd) reassoc nnan ninf nsz vp<[[VP2]]>, ir<%muladd> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP7]]> +; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds float, ir<%arrayidx>, ir<1> +; CHECK-NEXT: WIDEN ir<%l.a> = vp.load vp<[[VP8]]>, vp<%evl> +; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[VP7]]> +; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds float, ir<%arrayidx2>, ir<1> +; CHECK-NEXT: WIDEN ir<%l.b> = vp.load vp<[[VP9]]>, vp<%evl> +; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = fmul reassoc nnan ninf nsz ir<%l.a>, ir<%l.b> +; CHECK-NEXT: REDUCE ir<%muladd> = ir<%sum.07> + reassoc nnan ninf nsz vp.reduce.fadd (vp<[[VP10]]>, vp<%evl>) +; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP11]]>, vp<[[VP5]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]> +; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (fmuladd, in-loop) reassoc nnan ninf nsz ir<%muladd> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %loop ] + %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv + %l.a = load float, ptr %arrayidx, align 4 + %arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv + %l.b = load float, ptr %arrayidx2, align 4 + %muladd = tail call nnan ninf nsz reassoc float @llvm.fmuladd.f32(float %l.a, float %l.b, float %sum.07) + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, %n + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret float %muladd +} + +define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) { +; CHECK-LABEL: VPlan for loop in 'find_last_iv' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<%n> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP4:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP5:%[0-9]+]]> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (find-iv) ir<-9223372036854775808>, vp<[[VP8:%[0-9]+]]> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: EMIT-SCALAR vp<[[VP5]]> = zext vp<%evl> to i64 +; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP5]]> +; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<[[VP6]]> +; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i64, ir<%gep.a>, ir<1> +; CHECK-NEXT: WIDEN ir<%l.a> = vp.load vp<[[VP7]]>, vp<%evl> +; CHECK-NEXT: WIDEN ir<%cmp2> = icmp eq ir<%l.a>, ir<%start> +; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP8]]> = call llvm.vp.merge(ir<%cmp2>, ir<%iv>, ir<%rdx>, vp<%evl>) +; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP9]]>, vp<[[VP4]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP9]]> +; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP2]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (smax) vp<[[VP8]]> +; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp ne vp<[[VP11]]>, ir<-9223372036854775808> +; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = select vp<[[VP12]]>, vp<[[VP11]]>, ir<%start> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %inc, %loop ] + %rdx = phi i64 [ %start, %entry ], [ %cond, %loop ] + %gep.a = getelementptr inbounds i64, ptr %a, i64 %iv + %l.a = load i64, ptr %gep.a, align 8 + %cmp2 = icmp eq i64 %l.a, %start + %cond = select i1 %cmp2, i64 %iv, i64 %rdx + %inc = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %inc, %n + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret i64 %cond +} + +define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) { +; CHECK-LABEL: VPlan for loop in 'print_extended_reduction' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<%n> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1> +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, vp<[[VP8:%[0-9]+]]> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<%evl> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]> +; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i32, ir<%arrayidx>, ir<1> +; CHECK-NEXT: WIDEN ir<%load0> = vp.load vp<[[VP7]]>, vp<%evl> +; CHECK-NEXT: EXPRESSION vp<[[VP8]]> = vp<%evl> + reduce.add (ir<%load0> zext to i64, vp<%evl>) +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl> +; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP8]]> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: + %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ] + %rdx = phi i64 [ %rdx.next, %loop ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32, ptr %x, i32 %iv + %load0 = load i32, ptr %arrayidx, align 4 + %conv0 = zext i32 %load0 to i64 + %rdx.next = add nsw i64 %rdx, %conv0 + %iv.next = add nuw nsw i32 %iv, 1 + %exitcond = icmp eq i32 %iv.next, %n + br i1 %exitcond, label %exit, label %loop + +exit: + %r.0.lcssa = phi i64 [ %rdx.next, %loop ] + ret i64 %r.0.lcssa +} + +define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) { +; CHECK-LABEL: VPlan for loop in 'print_mulacc' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<%n> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1> +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, ir<%rdx.next> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<%evl> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]> +; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i64, ir<%arrayidx>, ir<1> +; CHECK-NEXT: WIDEN ir<%load0> = vp.load vp<[[VP7]]>, vp<%evl> +; CHECK-NEXT: CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP6]]> +; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds i64, ir<%arrayidx1>, ir<1> +; CHECK-NEXT: WIDEN ir<%load1> = vp.load vp<[[VP8]]>, vp<%evl> +; CHECK-NEXT: WIDEN ir<%mul> = mul nsw ir<%load0>, ir<%load1> +; CHECK-NEXT: REDUCE ir<%rdx.next> = ir<%rdx> + vp.reduce.add (ir<%mul>, vp<%evl>) +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl> +; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%rdx.next> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: + %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ] + %rdx = phi i64 [ %rdx.next, %loop ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i64, ptr %x, i32 %iv + %load0 = load i64, ptr %arrayidx, align 8 + %arrayidx1 = getelementptr inbounds i64, ptr %y, i32 %iv + %load1 = load i64, ptr %arrayidx1, align 8 + %mul = mul nsw i64 %load0, %load1 + %rdx.next = add nsw i64 %rdx, %mul + %iv.next = add nuw nsw i32 %iv, 1 + %exitcond = icmp eq i32 %iv.next, %n + br i1 %exitcond, label %exit, label %loop + +exit: + %r.0.lcssa = phi i64 [ %rdx.next, %loop ] + ret i64 %r.0.lcssa +} + +define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) { +; CHECK-LABEL: VPlan for loop in 'print_mulacc_extended' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<%n> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1> +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, vp<[[VP9:%[0-9]+]]> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<%evl> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]> +; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i16, ir<%arrayidx>, ir<1> +; CHECK-NEXT: WIDEN ir<%load0> = vp.load vp<[[VP7]]>, vp<%evl> +; CHECK-NEXT: CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP6]]> +; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds i16, ir<%arrayidx1>, ir<1> +; CHECK-NEXT: WIDEN ir<%load1> = vp.load vp<[[VP8]]>, vp<%evl> +; CHECK-NEXT: WIDEN-CAST ir<%conv0> = sext ir<%load0> to i32 +; CHECK-NEXT: WIDEN-CAST ir<%conv1> = sext ir<%load1> to i32 +; CHECK-NEXT: WIDEN ir<%mul> = mul nsw ir<%conv0>, ir<%conv1> +; CHECK-NEXT: EXPRESSION vp<[[VP9]]> = vp<%evl> + reduce.add (ir<%mul> sext to i64, vp<%evl>) +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl> +; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP9]]> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: + %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ] + %rdx = phi i64 [ %rdx.next, %loop ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i16, ptr %x, i32 %iv + %load0 = load i16, ptr %arrayidx, align 8 + %arrayidx1 = getelementptr inbounds i16, ptr %y, i32 %iv + %load1 = load i16, ptr %arrayidx1, align 8 + %conv0 = sext i16 %load0 to i32 + %conv1 = sext i16 %load1 to i32 + %mul = mul nsw i32 %conv0, %conv1 + %conv = sext i32 %mul to i64 + %rdx.next = add nsw i64 %rdx, %conv + %iv.next = add nuw nsw i32 %iv, 1 + %exitcond = icmp eq i32 %iv.next, %n + br i1 %exitcond, label %exit, label %loop + +exit: + %r.0.lcssa = phi i64 [ %rdx.next, %loop ] + ret i64 %r.0.lcssa +} + +define i32 @print_mulacc_negated(ptr %a, ptr %b) { +; CHECK-LABEL: VPlan for loop in 'print_mulacc_negated' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<1024> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1> +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[VP2]]>, ir<%add> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<1024>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]> +; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[VP7]]> +; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i8, ir<%gep.a>, ir<1> +; CHECK-NEXT: WIDEN ir<%load.a> = vp.load vp<[[VP8]]>, vp<%evl> +; CHECK-NEXT: WIDEN-CAST ir<%ext.a> = zext ir<%load.a> to i32 +; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[VP7]]> +; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, ir<%gep.b>, ir<1> +; CHECK-NEXT: WIDEN ir<%load.b> = vp.load vp<[[VP9]]>, vp<%evl> +; CHECK-NEXT: WIDEN-CAST ir<%ext.b> = zext ir<%load.b> to i32 +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%ext.b>, ir<%ext.a> +; CHECK-NEXT: WIDEN ir<%sub> = sub ir<0>, ir<%mul> +; CHECK-NEXT: REDUCE ir<%add> = ir<%accum> + vp.reduce.add (ir<%sub>, vp<%evl>) +; CHECK-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add nuw vp<[[VP10]]>, vp<[[VP5]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP10]]> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%add> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %accum = phi i32 [ 0, %entry ], [ %add, %loop ] + %gep.a = getelementptr i8, ptr %a, i64 %iv + %load.a = load i8, ptr %gep.a, align 1 + %ext.a = zext i8 %load.a to i32 + %gep.b = getelementptr i8, ptr %b, i64 %iv + %load.b = load i8, ptr %gep.b, align 1 + %ext.b = zext i8 %load.b to i32 + %mul = mul i32 %ext.b, %ext.a + %sub = sub i32 0, %mul + %add = add i32 %accum, %sub + %iv.next = add i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, 1024 + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret i32 %add +} + +define i32 @print_mulacc_extended_const(ptr %start, ptr %end) { +; CHECK-LABEL: VPlan for loop in 'print_mulacc_extended_const' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: EMIT vp<[[VP2]]> = EXPAND SCEV (1 + (-1 * (ptrtoaddr ptr %start to i64)) + (ptrtoaddr ptr %end to i64)) +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1> +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP3]]>, ir<%red.next> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP2]]>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]> +; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]> +; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, vp<%next.gep>, ir<1> +; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl> +; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = zext ir<%l> to i32 +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%l.ext>, ir<63> +; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + vp.reduce.add (ir<%mul>, vp<%evl>) +; CHECK-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP10]]>, vp<[[VP6]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP10]]> +; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%red.next> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: + %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] + %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] + %l = load i8, ptr %ptr.iv, align 1 + %l.ext = zext i8 %l to i32 + %mul = mul i32 %l.ext, 63 + %red.next = add i32 %red, %mul + %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1 + %ec = icmp eq ptr %ptr.iv, %end + br i1 %ec, label %exit, label %loop + +exit: + ret i32 %red.next +} + +define i32 @print_mulacc_extended_const_lhs(ptr %start, ptr %end) { +; CHECK-LABEL: VPlan for loop in 'print_mulacc_extended_const_lhs' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: EMIT vp<[[VP2]]> = EXPAND SCEV (1 + (-1 * (ptrtoaddr ptr %start to i64)) + (ptrtoaddr ptr %end to i64)) +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1> +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP3]]>, ir<%red.next> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP2]]>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]> +; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]> +; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, vp<%next.gep>, ir<1> +; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl> +; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = zext ir<%l> to i32 +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<63>, ir<%l.ext> +; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + vp.reduce.add (ir<%mul>, vp<%evl>) +; CHECK-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP10]]>, vp<[[VP6]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP10]]> +; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%red.next> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: + %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] + %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] + %l = load i8, ptr %ptr.iv, align 1 + %l.ext = zext i8 %l to i32 + %mul = mul i32 63, %l.ext + %red.next = add i32 %red, %mul + %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1 + %ec = icmp eq ptr %ptr.iv, %end + br i1 %ec, label %exit, label %loop + +exit: + ret i32 %red.next +} + +; Constants >= 128 cannot be treated as sign-extended, so the expression shouldn't extend 128 +define i32 @print_mulacc_not_extended_const(ptr %start, ptr %end) { +; CHECK-LABEL: VPlan for loop in 'print_mulacc_not_extended_const' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: EMIT vp<[[VP2]]> = EXPAND SCEV (1 + (-1 * (ptrtoaddr ptr %start to i64)) + (ptrtoaddr ptr %end to i64)) +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1> +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP3]]>, ir<%red.next> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP2]]>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]> +; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]> +; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, vp<%next.gep>, ir<1> +; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl> +; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = sext ir<%l> to i32 +; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = shl ir<%l.ext>, ir<7> +; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + vp.reduce.add (vp<[[VP10]]>, vp<%evl>) +; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP11]]>, vp<[[VP6]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]> +; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%red.next> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: + %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] + %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] + %l = load i8, ptr %ptr.iv, align 1 + %l.ext = sext i8 %l to i32 + %mul = mul i32 %l.ext, 128 + %red.next = add i32 %red, %mul + %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1 + %ec = icmp eq ptr %ptr.iv, %end + br i1 %ec, label %exit, label %loop + +exit: + %red.next.lcssa = phi i32 [ %red.next, %loop ] + ret i32 %red.next.lcssa +} + +define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) { +; CHECK-LABEL: VPlan for loop in 'print_ext_mulacc_extended_const' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: EMIT vp<[[VP2]]> = EXPAND SCEV (1 + (-1 * (ptrtoaddr ptr %start to i64)) + (ptrtoaddr ptr %end to i64)) +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1> +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP3]]>, vp<[[VP10:%[0-9]+]]> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP2]]>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]> +; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]> +; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, vp<%next.gep>, ir<1> +; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl> +; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = zext ir<%l> to i32 +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%l.ext>, ir<63> +; CHECK-NEXT: EXPRESSION vp<[[VP10]]> = vp<%evl> + reduce.add (ir<%mul> zext to i64, vp<%evl>) +; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP11]]>, vp<[[VP6]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]> +; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP10]]> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: + %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] + %red = phi i64 [ 0, %entry ], [ %red.next, %loop ] + %l = load i8, ptr %ptr.iv, align 1 + %l.ext = zext i8 %l to i32 + %mul = mul i32 %l.ext, 63 + %mul.ext = zext i32 %mul to i64 + %red.next = add i64 %red, %mul.ext + %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1 + %ec = icmp eq ptr %ptr.iv, %end + br i1 %ec, label %exit, label %loop + +exit: + ret i64 %red.next +} + +; Constants >= 128 cannot be treated as sign-extended, so the expression shouldn't extend 128 +define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) { +; CHECK-LABEL: VPlan for loop in 'print_ext_mulacc_not_extended_const' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: EMIT vp<[[VP2]]> = EXPAND SCEV (1 + (-1 * (ptrtoaddr ptr %start to i64)) + (ptrtoaddr ptr %end to i64)) +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1> +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP3]]>, vp<[[VP11:%[0-9]+]]> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP2]]>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]> +; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]> +; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, vp<%next.gep>, ir<1> +; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl> +; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = sext ir<%l> to i32 +; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = shl ir<%l.ext>, ir<7> +; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = vp<%evl> + reduce.add (vp<[[VP10]]> sext to i64, vp<%evl>) +; CHECK-NEXT: EMIT-SCALAR vp<[[VP12:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP12]]>, vp<[[VP6]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP12]]> +; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP11]]> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: + %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] + %red = phi i64 [ 0, %entry ], [ %red.next, %loop ] + %l = load i8, ptr %ptr.iv, align 1 + %l.ext = sext i8 %l to i32 + %mul = mul i32 %l.ext, 128 + %mul.ext = sext i32 %mul to i64 + %red.next = add i64 %red, %mul.ext + %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1 + %ec = icmp eq ptr %ptr.iv, %end + br i1 %ec, label %exit, label %loop + +exit: + %red.next.lcssa = phi i64 [ %red.next, %loop ] + ret i64 %red.next.lcssa +} + +; This reduce.add(ext(mul(ext(A), ext(B)))) can't be turned into an +; ExtMulAccReduction VPExpressionRecipe since the mul has two users. +; It can however be turned into an ExtendedReduction since that one doesn't +; modify the mul's operands. +define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) { +; CHECK-LABEL: VPlan for loop in 'print_ext_mul_two_uses' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: EMIT vp<[[VP2]]> = EXPAND SCEV (1 + %n) +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1> +; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext ir<%b> to i32 +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%conv>, ir<%conv> +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%res2> = phi (add) vp<[[VP3]]>, vp<[[VP7:%[0-9]+]]> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP2]]>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: EXPRESSION vp<[[VP7]]> = vp<%evl> + reduce.add (ir<%mul> zext to i64, vp<%evl>) +; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP8]]>, vp<[[VP6]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP8]]> +; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP7]]> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %res1 = phi i64 [ 0, %entry ], [ %load.ext.ext, %loop ] + %res2 = phi i64 [ 0, %entry ], [ %add, %loop ] + %load = load i16, ptr %a, align 2 + %iv.next = add i64 %iv, 1 + %conv = sext i16 %b to i32 + %mul = mul i32 %conv, %conv + %mul.ext = zext i32 %mul to i64 + %add = add i64 %res2, %mul.ext + %second_use = or i32 %mul, %c ; this value is otherwise unused, but that's sufficient for the test + %load.ext = sext i16 %load to i32 + %load.ext.ext = sext i32 %load.ext to i64 + %exitcond740.not = icmp eq i64 %iv, %n + br i1 %exitcond740.not, label %exit, label %loop + +exit: + ret i64 %add +} + +define i32 @print_umax_reduction(ptr %y) { +; CHECK-LABEL: VPlan for loop in 'print_umax_reduction' +; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<100> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb<entry>: +; CHECK-NEXT: Successor(s): scalar.ph, vector.ph +; CHECK-EMPTY: +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: <x1> vector loop: { +; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = CANONICAL-IV +; CHECK-EMPTY: +; CHECK-NEXT: vector.body: +; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP4:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (umax) ir<0>, vp<[[VP8:%[0-9]+]]> +; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<100>, vector.ph ], [ vp<%avl.next>, vector.body ] +; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl> +; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP5]]> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%y>, vp<[[VP6]]> +; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep>, ir<1> +; CHECK-NEXT: WIDEN ir<%lv> = vp.load vp<[[VP7]]>, vp<%evl> +; CHECK-NEXT: WIDEN-INTRINSIC ir<%red.next> = call llvm.umax(ir<%lv>, ir<%red>) +; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP8]]> = call llvm.vp.merge(ir<true>, ir<%red.next>, ir<%red>, vp<%evl>) +; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = zext vp<%evl> to i64 +; CHECK-NEXT: EMIT vp<%current.iteration.next> = add nuw vp<[[VP9]]>, vp<[[VP4]]> +; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP9]]> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP2]]>, vp<[[VP0]]> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): middle.block +; CHECK-EMPTY: +; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (umax) vp<[[VP8]]> +; CHECK-NEXT: Successor(s): ir-bb<exit> +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] + %gep = getelementptr inbounds i32, ptr %y, i64 %iv + %lv = load i32, ptr %gep, align 4 + %red.next = call i32 @llvm.umax(i32 %lv, i32 %red) + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 100 + br i1 %ec, label %exit, label %loop + +exit: + ret i32 %red.next +}