llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck %s

target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

; Test case for https://github.com/llvm/llvm-project/issues/111042.
define void @replicate_udiv_with_only_first_lane_used(i32 %x, ptr %dst, i64 %d) {
; CHECK-LABEL: define void @replicate_udiv_with_only_first_lane_used(
; CHECK-SAME: i32 [[X:%.*]], ptr [[DST:%.*]], i64 [[D:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X]], 10
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    store i16 0, ptr [[DST]], align 2
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[SCALAR_PH:.*]]
; CHECK:       [[SCALAR_PH]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 96, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    br i1 true, label %[[LOOP_LATCH]], label %[[ELSE:.*]]
; CHECK:       [[ELSE]]:
; CHECK-NEXT:    [[DIV_I:%.*]] = udiv i64 99, [[D]]
; CHECK-NEXT:    br label %[[LOOP_LATCH]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[RETVAL_0_I:%.*]] = phi i64 [ [[DIV_I]], %[[ELSE]] ], [ 0, %[[LOOP_HEADER]] ]
; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[DST]], i64 [[RETVAL_0_I]]
; CHECK-NEXT:    store i16 0, ptr [[GEP]], align 2
; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[IV_NEXT]], 101
; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP_HEADER]], label %[[EXIT:.*]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  %c = icmp eq i32 %x, 10
  br label %loop.header

loop.header:                                         ; preds = %loop.latch, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  br i1 true, label %loop.latch, label %else

else:
  %div.i = udiv i64 99, %d
  br label %loop.latch

loop.latch:
  %retval.0.i = phi i64 [ %div.i, %else ], [ 0, %loop.header ]
  %gep = getelementptr i16, ptr %dst, i64 %retval.0.i
  store i16 0, ptr %gep, align 2
  %iv.next = add i64 %iv, 1
  %cmp = icmp ult i64 %iv.next, 101
  br i1 %cmp, label %loop.header, label %exit

exit:
  ret void
}

; Variant where ptradd cannot be const-folded.
define void @replicate_udiv_with_only_first_lane_used2(i32 %x, ptr %dst, i64 %d) {
; CHECK-LABEL: define void @replicate_udiv_with_only_first_lane_used2(
; CHECK-SAME: i32 [[X:%.*]], ptr [[DST:%.*]], i64 [[D:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X]], 10
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UDIV_CONTINUE14:.*]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
; CHECK-NEXT:    br i1 [[TMP1]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; CHECK:       [[PRED_UDIV_IF]]:
; CHECK-NEXT:    [[TMP2:%.*]] = udiv i64 99, [[D]]
; CHECK-NEXT:    br label %[[PRED_UDIV_CONTINUE]]
; CHECK:       [[PRED_UDIV_CONTINUE]]:
; CHECK-NEXT:    [[TMP3:%.*]] = phi i64 [ poison, %[[VECTOR_BODY]] ], [ [[TMP2]], %[[PRED_UDIV_IF]] ]
; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2:.*]]
; CHECK:       [[PRED_UDIV_IF1]]:
; CHECK-NEXT:    [[TMP6:%.*]] = udiv i64 99, [[D]]
; CHECK-NEXT:    br label %[[PRED_UDIV_CONTINUE2]]
; CHECK:       [[PRED_UDIV_CONTINUE2]]:
; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
; CHECK-NEXT:    br i1 [[TMP9]], label %[[PRED_UDIV_IF3:.*]], label %[[PRED_UDIV_CONTINUE4:.*]]
; CHECK:       [[PRED_UDIV_IF3]]:
; CHECK-NEXT:    [[TMP10:%.*]] = udiv i64 99, [[D]]
; CHECK-NEXT:    br label %[[PRED_UDIV_CONTINUE4]]
; CHECK:       [[PRED_UDIV_CONTINUE4]]:
; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
; CHECK-NEXT:    br i1 [[TMP13]], label %[[PRED_UDIV_IF5:.*]], label %[[PRED_UDIV_CONTINUE6:.*]]
; CHECK:       [[PRED_UDIV_IF5]]:
; CHECK-NEXT:    [[TMP14:%.*]] = udiv i64 99, [[D]]
; CHECK-NEXT:    br label %[[PRED_UDIV_CONTINUE6]]
; CHECK:       [[PRED_UDIV_CONTINUE6]]:
; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
; CHECK-NEXT:    br i1 [[TMP17]], label %[[PRED_UDIV_IF7:.*]], label %[[PRED_UDIV_CONTINUE8:.*]]
; CHECK:       [[PRED_UDIV_IF7]]:
; CHECK-NEXT:    [[TMP18:%.*]] = udiv i64 99, [[D]]
; CHECK-NEXT:    br label %[[PRED_UDIV_CONTINUE8]]
; CHECK:       [[PRED_UDIV_CONTINUE8]]:
; CHECK-NEXT:    [[TMP15:%.*]] = phi i64 [ poison, %[[PRED_UDIV_CONTINUE6]] ], [ [[TMP18]], %[[PRED_UDIV_IF7]] ]
; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
; CHECK-NEXT:    br i1 [[TMP21]], label %[[PRED_UDIV_IF9:.*]], label %[[PRED_UDIV_CONTINUE10:.*]]
; CHECK:       [[PRED_UDIV_IF9]]:
; CHECK-NEXT:    [[TMP22:%.*]] = udiv i64 99, [[D]]
; CHECK-NEXT:    br label %[[PRED_UDIV_CONTINUE10]]
; CHECK:       [[PRED_UDIV_CONTINUE10]]:
; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
; CHECK-NEXT:    br i1 [[TMP25]], label %[[PRED_UDIV_IF11:.*]], label %[[PRED_UDIV_CONTINUE12:.*]]
; CHECK:       [[PRED_UDIV_IF11]]:
; CHECK-NEXT:    [[TMP26:%.*]] = udiv i64 99, [[D]]
; CHECK-NEXT:    br label %[[PRED_UDIV_CONTINUE12]]
; CHECK:       [[PRED_UDIV_CONTINUE12]]:
; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
; CHECK-NEXT:    br i1 [[TMP29]], label %[[PRED_UDIV_IF13:.*]], label %[[PRED_UDIV_CONTINUE14]]
; CHECK:       [[PRED_UDIV_IF13]]:
; CHECK-NEXT:    [[TMP30:%.*]] = udiv i64 99, [[D]]
; CHECK-NEXT:    br label %[[PRED_UDIV_CONTINUE14]]
; CHECK:       [[PRED_UDIV_CONTINUE14]]:
; CHECK-NEXT:    [[TMP45:%.*]] = select i1 [[C]], i64 0, i64 [[TMP3]]
; CHECK-NEXT:    [[TMP47:%.*]] = select i1 [[C]], i64 0, i64 [[TMP15]]
; CHECK-NEXT:    [[TMP46:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP45]]
; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP47]]
; CHECK-NEXT:    store i16 0, ptr [[TMP46]], align 2
; CHECK-NEXT:    store i16 0, ptr [[TMP48]], align 2
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[SCALAR_PH:.*]]
; CHECK:       [[SCALAR_PH]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 96, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_LATCH]], label %[[ELSE:.*]]
; CHECK:       [[ELSE]]:
; CHECK-NEXT:    [[DIV_I:%.*]] = udiv i64 99, [[D]]
; CHECK-NEXT:    br label %[[LOOP_LATCH]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[RETVAL_0_I:%.*]] = phi i64 [ [[DIV_I]], %[[ELSE]] ], [ 0, %[[LOOP_HEADER]] ]
; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[DST]], i64 [[RETVAL_0_I]]
; CHECK-NEXT:    store i16 0, ptr [[GEP]], align 2
; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[IV_NEXT]], 101
; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP_HEADER]], label %[[EXIT:.*]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  %c = icmp eq i32 %x, 10
  br label %loop.header

loop.header:                                         ; preds = %loop.latch, %entry
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  br i1 %c, label %loop.latch, label %else

else:
  %div.i = udiv i64 99, %d
  br label %loop.latch

loop.latch:
  %retval.0.i = phi i64 [ %div.i, %else ], [ 0, %loop.header ]
  %gep = getelementptr i16, ptr %dst, i64 %retval.0.i
  store i16 0, ptr %gep, align 2
  %iv.next = add i64 %iv, 1
  %cmp = icmp ult i64 %iv.next, 101
  br i1 %cmp, label %loop.header, label %exit

exit:
  ret void
}

define float @uniform_load_replicating_select(ptr %A, ptr %B, i64 %1) {
; CHECK-LABEL: define float @uniform_load_replicating_select(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[TMP0:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 1
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 8
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 8
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 5
; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 6
; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 7
; CHECK-NEXT:    [[TMP6:%.*]] = load float, ptr [[A]], align 4
; CHECK-NEXT:    [[TMP10:%.*]] = fcmp ogt float [[TMP6]], 0.000000e+00
; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP2]]
; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP3]]
; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP4]]
; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP5]]
; CHECK-NEXT:    [[TMP19:%.*]] = select i1 [[TMP10]], ptr [[A]], ptr [[TMP15]]
; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP10]], ptr [[A]], ptr [[TMP16]]
; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP10]], ptr [[A]], ptr [[TMP17]]
; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP10]], ptr [[A]], ptr [[TMP18]]
; CHECK-NEXT:    [[TMP24:%.*]] = load float, ptr [[TMP19]], align 4
; CHECK-NEXT:    [[TMP25:%.*]] = load float, ptr [[TMP20]], align 4
; CHECK-NEXT:    [[TMP26:%.*]] = load float, ptr [[TMP21]], align 4
; CHECK-NEXT:    [[TMP27:%.*]] = load float, ptr [[TMP22]], align 4
; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <4 x float> poison, float [[TMP24]], i32 0
; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <4 x float> [[TMP28]], float [[TMP25]], i32 1
; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <4 x float> [[TMP29]], float [[TMP26]], i32 2
; CHECK-NEXT:    [[TMP31:%.*]] = insertelement <4 x float> [[TMP30]], float [[TMP27]], i32 3
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP10]], <4 x float> splat (float 1.000000e+01), <4 x float> splat (float 1.000000e+00)
; CHECK-NEXT:    [[TMP36:%.*]] = fdiv <4 x float> splat (float 4.000000e+00), [[TMP31]]
; CHECK-NEXT:    [[TMP33:%.*]] = call <4 x float> @llvm.pow.v4f32(<4 x float> [[TMP32]], <4 x float> [[TMP36]])
; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <4 x float> [[TMP33]], i32 3
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT:    br label %[[LOOP:.*]]
; CHECK:       [[LOOP]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[A]], align 4
; CHECK-NEXT:    [[C:%.*]] = fcmp ogt float [[L]], 0.000000e+00
; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
; CHECK-NEXT:    [[SEL_PTR:%.*]] = select i1 [[C]], ptr [[A]], ptr [[GEP_B]]
; CHECK-NEXT:    [[BASE:%.*]] = select i1 [[C]], float 1.000000e+01, float 1.000000e+00
; CHECK-NEXT:    [[L_2:%.*]] = load float, ptr [[SEL_PTR]], align 4
; CHECK-NEXT:    [[DIV:%.*]] = fdiv float 4.000000e+00, [[L_2]]
; CHECK-NEXT:    [[POW:%.*]] = tail call float @llvm.pow.f32(float [[BASE]], float [[DIV]])
; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[TMP0]]
; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[POW_LCSSA:%.*]] = phi float [ [[POW]], %[[LOOP]] ], [ [[TMP35]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT:    ret float [[POW_LCSSA]]
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
  %l = load float, ptr %A, align 4
  %c = fcmp ogt float %l, 0.000000e+00
  %gep.B = getelementptr inbounds float, ptr %B, i64 %iv
  %sel.ptr = select i1 %c, ptr %A, ptr %gep.B
  %base = select i1 %c, float 10.000000e+00, float 1.000000e+00
  %l.2 = load float, ptr %sel.ptr, align 4
  %div = fdiv float 4.000000e+00, %l.2
  %pow = tail call float @llvm.pow.f32(float %base, float %div)
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv, %1
  br i1 %ec, label %exit, label %loop

exit:
  ret float %pow
}

declare float @llvm.pow.f32(float, float)