[VPlan] Allow early exit loops with side effects by masking in the vector loop. When all blocks with side effects are dominated by every early exit, we can vectorize the loop if we mask off any lanes past the exit. We can do this with control flow: ```mermaid flowchart TD header --> exiting exiting -->| exit mask | sideeffects --> latch --> latch.split exiting -->| !exit mask | latch.split ``` For every early exiting block we add a branch to the latch on `!mask.beforefirst(exitcond)`. This is implemented in UncountableExitStyle::ReadOnly which is now renamed to MaskedHandleExitInVectorLoop. This is a complementary exit style to MaskedHandleExitInScalarLoop. If any of the side effects aren't dominated by the early exit it still falls back to the scalar loop style, which is why none of the existing early exit side effect tests changed. Masking in loop is profitable for RISC-V and I'm assuming it's also profitable for AArch64 too, but if not then we can add a TTI hook for a preferred exit style.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index c9b2a14..01ca7e7 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6268,7 +6268,8 @@ return nullptr; // TODO: Check target preference for style. - UncountableExitStyle EEStyle = UncountableExitStyle::ReadOnly; + UncountableExitStyle EEStyle = + UncountableExitStyle::MaskedHandleExitInVectorLoop; if (!RUN_VPLAN_PASS(VPlanTransforms::handleUncountableEarlyExits, *VPlan0, ORE, OrigLoop, PSE, *DT, Legal->getAssumptionCache(), EEStyle)) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 5d89141..8331fb8 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -79,10 +79,11 @@ /// Different methods of handling early exits. /// enum class UncountableExitStyle { - /// No side effects to worry about, so we can process any uncountable exits - /// in the loop and branch either to the middle block if the trip count was - /// reached, or an early exitblock to determine which exit was taken. - ReadOnly, + /// Any side effects come after all the uncountable exits and are masked off, + /// so we can process any uncountable exits in the loop and branch either to + /// the middle block if the trip count was reached, or an early exitblock to + /// determine which exit was taken. + MaskedHandleExitInVectorLoop, /// All memory operations other than the load(s) required to determine whether /// an uncountable exit occurre will be masked based on that condition. If an /// uncountable exit is taken, then all lanes before the exiting lane will
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index 49b8104..c73d81f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1272,7 +1272,9 @@ continue; auto *VPI = dyn_cast<VPInstruction>(&R); if (!VPI || VPI->getOpcode() != Instruction::Load) { - assert(!R.mayReadFromMemory() && "unexpected recipe reading memory"); + assert((!R.mayReadFromMemory() || + (VPI && VPI->getOpcode() == Instruction::Store)) && + "unexpected recipe reading memory"); continue; }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 88912a5..b09bfe4 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3554,50 +3554,59 @@ // Returns true if the loop contains stores, the only writes legality accepts in // early-exit loops. -static bool loopHasStores(VPBasicBlock *HeaderVPBB) { - return any_of(vp_rpo_plain_cfg_loop_body(HeaderVPBB), [](VPBasicBlock *VPBB) { - return any_of(*VPBB, [VPBB](VPRecipeBase &R) { - (void)VPBB; - if (match(&R, m_Intrinsic<Intrinsic::pseudoprobe>())) - return false; - if (!R.mayWriteToMemory()) { - assert((!R.mayHaveSideEffects() || &R == VPBB->getTerminator() || - R.isSafeToSpeculativelyExecute()) && - "legality only accepts recipes with side effects that are " - "safe to speculate"); - return false; - } - assert(match(&R, m_VPInstruction<Instruction::Store>()) && - "legality only accepts stores as writes in early-exit loops"); - return true; - }); - }); +static SmallVector<VPBasicBlock *> blocksWithStores(VPBasicBlock *HeaderVPBB) { + return to_vector(make_filter_range( + vp_rpo_plain_cfg_loop_body(HeaderVPBB), [](VPBasicBlock *VPBB) { + return any_of(*VPBB, [VPBB](VPRecipeBase &R) { + (void)VPBB; + if (match(&R, m_Intrinsic<Intrinsic::pseudoprobe>())) + return false; + if (!R.mayWriteToMemory()) { + assert((!R.mayHaveSideEffects() || &R == VPBB->getTerminator() || + R.isSafeToSpeculativelyExecute()) && + "legality only accepts recipes with side effects that are " + "safe to speculate"); + return false; + } + assert(match(&R, m_VPInstruction<Instruction::Store>()) && + "legality only accepts stores as writes in early-exit loops"); + return true; + }); + })); } bool VPlanTransforms::handleUncountableEarlyExits( VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop, PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style) { -#ifndef NDEBUG VPDominatorTree VPDT(Plan); -#endif auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan); auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan); - if (loopHasStores(HeaderVPBB)) - Style = UncountableExitStyle::MaskedHandleExitInScalarLoop; + // If all side effects are dominated by the early exits, we can mask in the + // vector loop. Otherwise we need to handle side effects in the scalar loop. + // TODO: Legality only allows side effects that the scalar loop style can + // handle, relax it for side effects after all the early exits. + SmallVector<VPBasicBlock *> SideEffectVPBBs = blocksWithStores(HeaderVPBB); + if (!SideEffectVPBBs.empty()) { + auto EarlyExits = vputils::getEarlyExits(Plan, MiddleVPBB); + for (auto [EarlyExitingVPBB, _] : EarlyExits) + for (VPBasicBlock *VPBB : SideEffectVPBBs) + if (!VPDT.properlyDominates(EarlyExitingVPBB, VPBB)) + Style = UncountableExitStyle::MaskedHandleExitInScalarLoop; + } - // Dereferenceability is checked separately for uncountable exit loops with - // stores, as only the loads contributing to the exit condition need to - // be checked. - if (Style == UncountableExitStyle::ReadOnly && + if (Style == UncountableExitStyle::MaskedHandleExitInVectorLoop && !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC)) { - reportVectorizationFailure( - "Auto-vectorization of early exit loops with potentially " - "faulting loads is not supported", - "EarlyExitFaultingLoads", ORE, TheLoop); - return false; + if (SideEffectVPBBs.empty()) { + reportVectorizationFailure( + "Auto-vectorization of early exit loops with potentially " + "faulting loads is not supported", + "EarlyExitFaultingLoads", ORE, TheLoop); + return false; + } + Style = UncountableExitStyle::MaskedHandleExitInScalarLoop; } VPBuilder LatchBuilder(LatchVPBB->getTerminator()); @@ -3649,7 +3658,7 @@ }); } - if (Style == UncountableExitStyle::ReadOnly) { + if (Style == UncountableExitStyle::MaskedHandleExitInVectorLoop) { // Lanes past the exit can be poison that the scalar loop never branched on, // so freeze them to prevent branching on poison. for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB)) { @@ -3659,6 +3668,49 @@ Term->setOperand(0, VPBuilder(Term).createFreeze(Term->getOperand(0))); } + if (!SideEffectVPBBs.empty()) { + // Split the latch in case it has recipes with side effects. + LatchVPBB = LatchVPBB->splitAt(LatchVPBB->getTerminator()->getIterator()); + LatchBuilder.setInsertPoint(LatchVPBB->getTerminator()); + + // Branch to the latch to skip any side effects past the exiting lane. + for (EarlyExitInfo &Exit : Exits) { + VPBasicBlock *EarlyExitingVPBB = Exit.EarlyExitingVPBB; + VPBuilder ExitingBuilder(EarlyExitingVPBB->getTerminator()); + VPValue *ExitMask = ExitingBuilder.createWidenIntrinsic( + Intrinsic::mask_beforefirst, + ExitingBuilder.createFreeze(ExitingBuilder.createNaryOp( + VPInstruction::MaskedCond, Exit.CondToExit)), + Type::getInt1Ty(Plan.getContext())); + EarlyExitingVPBB->getTerminator()->setOperand(0, ExitMask); + VPBlockUtils::connectBlocks(EarlyExitingVPBB, LatchVPBB); + } + + // The latch's exit condition may be poison on lanes past the exit, freeze + // it. + VPRecipeBase *LatchTerm = LatchVPBB->getTerminator(); + LatchTerm->setOperand( + 0, VPBuilder(LatchTerm).createFreeze(LatchTerm->getOperand(0))); + + // If we took the exiting branch then any uses in the latch and beyond are + // poison. + VPDT.recalculate(Plan); + for (VPBasicBlock *VPBB : {LatchVPBB, MiddleVPBB}) { + for (VPRecipeBase &R : + make_range(VPBB->getFirstNonPhi(), VPBB->end())) { + for (auto [I, Op] : enumerate(R.operands())) { + VPRecipeBase *DefR = Op->getDefiningRecipe(); + if (!DefR || VPDT.dominates(DefR->getParent(), VPBB)) + continue; + DenseMap<VPBasicBlock *, VPValue *> Defs = { + {HeaderVPBB, Plan.getPoison(Op->getScalarType())}}; + Defs[DefR->getParent()] = Op; + R.setOperand(I, vputils::reconstructSSA(LatchVPBB, Defs)); + } + } + } + } + // An exit's condition is false on any path that didn't go through its // exiting block. for (EarlyExitInfo &Exit : Exits) { @@ -3816,7 +3868,8 @@ ExitIRI->addIncoming(NewIncoming); } - EarlyExitingVPBB->getTerminator()->eraseFromParent(); + if (EarlyExitingVPBB->getNumSuccessors() == 2) + EarlyExitingVPBB->getTerminator()->eraseFromParent(); VPBlockUtils::disconnectBlocks(EarlyExitingVPBB, EarlyExitVPBB); VPBlockUtils::connectBlocks(VectorEarlyExitVPBB, EarlyExitVPBB); }
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll index 573ee73..8ba6d5f 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -1277,7 +1277,7 @@ ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]] ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ] ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP2]], align 2 ; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500) @@ -1291,15 +1291,16 @@ ; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP4]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]] -; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP4]], i1 false) -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]] -; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20 +; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK1]]: +; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 20, [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]] +; CHECK: [[VECTOR_EARLY_EXIT]]: +; CHECK-NEXT: br label %[[EXIT]] ; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK1]] ], [ 0, %[[ENTRY]] ] ; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] ; CHECK: [[LOOP_HEADER]]: ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll index 6fcbe46..6f167b3 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/early_exit_with_stores.ll
@@ -12,7 +12,7 @@ ; CHECK: [[LOOP_HEADER]]: ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ] +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[INDEX_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ] ; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[GEP_SRC]], align 2 ; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500) @@ -54,7 +54,7 @@ ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]] ; CHECK: [[PRED_STORE_CONTINUE4]]: ; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], i64 3 -; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]] +; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]] ; CHECK: [[PRED_STORE_IF5]]: ; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[IV]], 3 ; CHECK-NEXT: [[TMP23:%.*]] = shl i64 [[TMP22]], 1 @@ -68,14 +68,15 @@ ; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4 ; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 -; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]] -; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP27]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false) -; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[IV]], [[TMP3]] -; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[TMP30]], 20 -; CHECK-NEXT: br i1 [[TMP31]], [[EXIT:label %.*]], label %[[SCALAR_PH1:.*]] +; CHECK-NEXT: br i1 [[TMP28]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: [[SCALAR_PH1]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[VECTOR_EARLY_EXIT]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void ; entry: br label %loop.header @@ -168,7 +169,7 @@ ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4 ; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 ; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]] -; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false) ; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[IV]], [[TMP3]]
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll index 139c999..d4f5ef9 100644 --- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll +++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -1564,7 +1564,7 @@ ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ] ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2 ; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500) @@ -1577,29 +1577,13 @@ ; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 -; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] -; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]] +; CHECK: [[VECTOR_BODY_INTERIM]]: +; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false) -; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]] -; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 20 -; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: br label %[[LOOP:.*]] -; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP10]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] -; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]] -; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2 -; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500 -; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_LATCH]] -; CHECK: [[LOOP_LATCH]]: -; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]] -; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2 -; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1 -; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2 -; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 -; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20 -; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]] +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[VECTOR_EARLY_EXIT]]: +; CHECK-NEXT: br label %[[EXIT]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; @@ -1629,32 +1613,49 @@ define void @conditional_store_after_two_exits(ptr align 2 dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred, ptr align 2 dereferenceable(40) readonly %pred2) { ; CHECK-LABEL: define void @conditional_store_after_two_exits( ; CHECK-SAME: ptr noalias align 2 dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED2:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: br label %[[LOOP:.*]] -; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] -; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]] -; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2 -; CHECK-NEXT: [[EE2_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED2]], i64 [[IV]] -; CHECK-NEXT: [[EE2_VAL:%.*]] = load i16, ptr [[EE2_ADDR]], align 2 -; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500 -; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[LOOP_2:.*]] -; CHECK: [[LOOP_2]]: -; CHECK-NEXT: [[EE2_COND:%.*]] = icmp eq i16 [[EE2_VAL]], 0 -; CHECK-NEXT: br i1 [[EE2_COND]], label %[[EXIT_2:.*]], label %[[LOOP_3:.*]] -; CHECK: [[LOOP_3]]: -; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]] -; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2 -; CHECK-NEXT: [[ST_COND:%.*]] = icmp slt i16 [[DATA]], 0 -; CHECK-NEXT: br i1 [[ST_COND]], label %[[STORE:.*]], label %[[LOOP_LATCH]] -; CHECK: [[STORE]]: -; CHECK-NEXT: [[INC:%.*]] = add i16 [[DATA]], 1 -; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2 -; CHECK-NEXT: br label %[[LOOP_LATCH]] -; CHECK: [[LOOP_LATCH]]: -; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 -; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20 -; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]] +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2 +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED2]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2 +; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500) +; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]] +; CHECK-NEXT: [[TMP4:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i16> [[WIDE_LOAD1]], zeroinitializer +; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = call <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> [[TMP7]]) +; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP10]], <4 x i1> [[TMP9]], <4 x i16> poison) +; CHECK-NEXT: [[TMP11:%.*]] = icmp slt <4 x i16> [[WIDE_MASKED_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP11]] +; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> [[TMP12]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP14:%.*]] = add <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1) +; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP14]], ptr align 2 [[TMP10]], <4 x i1> [[TMP13]]) +; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP15]] +; CHECK-NEXT: [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 +; CHECK-NEXT: br i1 [[TMP17]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]] +; CHECK: [[VECTOR_BODY_INTERIM]]: +; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]: +; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP16]], i1 false) +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]] +; CHECK-NEXT: br i1 [[TMP19]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]] +; CHECK: [[VECTOR_EARLY_EXIT_1]]: +; CHECK-NEXT: br label %[[EXIT_2:.*]] +; CHECK: [[VECTOR_EARLY_EXIT_0]]: +; CHECK-NEXT: br label %[[EXIT]] ; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; CHECK: [[EXIT_2]]: @@ -1707,7 +1708,7 @@ ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ] ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2 ; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500) @@ -1720,34 +1721,16 @@ ; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20 -; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] -; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]] +; CHECK: [[VECTOR_BODY_INTERIM]]: +; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false) -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] -; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20 -; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: br label %[[LOOP:.*]] -; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP11]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] -; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]] -; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2 -; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500 -; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[LOOP_STORE:.*]] -; CHECK: [[LOOP_STORE]]: -; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]] -; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2 -; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1 -; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2 -; CHECK-NEXT: br label %[[LOOP_LATCH]] -; CHECK: [[LOOP_LATCH]]: -; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 -; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20 -; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP23:![0-9]+]] +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i16> [[TMP5]], i64 3 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[VECTOR_EARLY_EXIT]]: +; CHECK-NEXT: br label %[[EXIT]] ; CHECK: [[EXIT]]: -; CHECK-NEXT: [[RES:%.*]] = phi i16 [ -1, %[[LOOP]] ], [ [[INC]], %[[LOOP_LATCH]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[RES:%.*]] = phi i16 [ -1, %[[VECTOR_EARLY_EXIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ] ; CHECK-NEXT: ret i16 [[RES]] ; entry:
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp index e006ed7..aaf677b 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -578,8 +578,9 @@ BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); // Build a plain CFG VPlan with BranchOnTwoConds as the latch terminator // (3 successors), without wrapping blocks in loop regions. - auto Plan = buildVPlan(LoopHeader, UncountableExitStyle::ReadOnly, - /*CreateLoopRegions=*/false); + auto Plan = + buildVPlan(LoopHeader, UncountableExitStyle::MaskedHandleExitInVectorLoop, + /*CreateLoopRegions=*/false); auto *MiddleVPBB = cast<VPBasicBlock>(Plan->getScalarPreheader()->getPredecessors()[0]);