llvm/test/Transforms/LoopVectorize/X86/compress-idioms.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^(scalar.ph)|(for.body):" --version 6
; RUN: opt < %s -mtriple=x86_64-- -mcpu=x86-64-v4 -passes=loop-vectorize -S | FileCheck %s --check-prefix=CHECK-V4
; RUN: opt < %s -mtriple=x86_64-- -mcpu=icelake-server -passes=loop-vectorize -S | FileCheck %s --check-prefix=CHECK-ICELAKE
; RUN: opt < %s -mtriple=x86_64-- -mcpu=znver4 -passes=loop-vectorize -S | FileCheck %s --check-prefix=CHECK-ZNVER4

define void @compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
; CHECK-V4-LABEL: define void @compress_store(
; CHECK-V4-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-V4-NEXT:  [[ENTRY:.*:]]
; CHECK-V4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-V4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[EXIT:.*]], label %[[VECTOR_PH:.*]]
; CHECK-V4:       [[VECTOR_PH]]:
; CHECK-V4-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 7
; CHECK-V4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-V4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[C]], i64 0
; CHECK-V4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
; CHECK-V4-NEXT:    br label %[[FOR_BODY:.*]]
; CHECK-V4:       [[FOR_BODY]]:
; CHECK-V4-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-V4-NEXT:    [[IDX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[FOR_BODY]] ]
; CHECK-V4-NEXT:    [[SRC_PTR:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
; CHECK-V4-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[SRC_PTR]], align 4
; CHECK-V4-NEXT:    [[TMP2:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-V4-NEXT:    [[DST_PTR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IDX]]
; CHECK-V4-NEXT:    call void @llvm.masked.compressstore.v8i32.p0(<8 x i32> [[WIDE_LOAD]], ptr align 4 [[DST_PTR]], <8 x i1> [[TMP2]])
; CHECK-V4-NEXT:    [[TMP4:%.*]] = zext <8 x i1> [[TMP2]] to <8 x i64>
; CHECK-V4-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP4]])
; CHECK-V4-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[IDX]], [[TMP5]]
; CHECK-V4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
; CHECK-V4-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-V4-NEXT:    br i1 [[TMP6]], label %[[FOR_INC:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-V4:       [[FOR_INC]]:
; CHECK-V4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-V4-NEXT:    br i1 [[CMP_N]], [[EXIT1:label %.*]], label %[[EXIT]]
; CHECK-V4:       [[EXIT]]:
;
; CHECK-ICELAKE-LABEL: define void @compress_store(
; CHECK-ICELAKE-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-ICELAKE-NEXT:  [[ENTRY:.*:]]
; CHECK-ICELAKE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-ICELAKE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[EXIT:.*]], label %[[VECTOR_PH:.*]]
; CHECK-ICELAKE:       [[VECTOR_PH]]:
; CHECK-ICELAKE-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 7
; CHECK-ICELAKE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-ICELAKE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[C]], i64 0
; CHECK-ICELAKE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
; CHECK-ICELAKE-NEXT:    br label %[[FOR_BODY:.*]]
; CHECK-ICELAKE:       [[FOR_BODY]]:
; CHECK-ICELAKE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-ICELAKE-NEXT:    [[IDX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[FOR_BODY]] ]
; CHECK-ICELAKE-NEXT:    [[SRC_PTR:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
; CHECK-ICELAKE-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[SRC_PTR]], align 4
; CHECK-ICELAKE-NEXT:    [[TMP2:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-ICELAKE-NEXT:    [[DST_PTR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IDX]]
; CHECK-ICELAKE-NEXT:    call void @llvm.masked.compressstore.v8i32.p0(<8 x i32> [[WIDE_LOAD]], ptr align 4 [[DST_PTR]], <8 x i1> [[TMP2]])
; CHECK-ICELAKE-NEXT:    [[TMP4:%.*]] = zext <8 x i1> [[TMP2]] to <8 x i64>
; CHECK-ICELAKE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP4]])
; CHECK-ICELAKE-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[IDX]], [[TMP5]]
; CHECK-ICELAKE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
; CHECK-ICELAKE-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-ICELAKE-NEXT:    br i1 [[TMP6]], label %[[FOR_INC:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-ICELAKE:       [[FOR_INC]]:
; CHECK-ICELAKE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-ICELAKE-NEXT:    br i1 [[CMP_N]], [[EXIT1:label %.*]], label %[[EXIT]]
; CHECK-ICELAKE:       [[EXIT]]:
;
; CHECK-ZNVER4-LABEL: define void @compress_store(
; CHECK-ZNVER4-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-ZNVER4-NEXT:  [[ENTRY:.*:]]
; CHECK-ZNVER4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
; CHECK-ZNVER4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-ZNVER4:       [[VECTOR_PH]]:
; CHECK-ZNVER4-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 15
; CHECK-ZNVER4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-ZNVER4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[C]], i64 0
; CHECK-ZNVER4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
; CHECK-ZNVER4-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-ZNVER4:       [[VECTOR_BODY]]:
; CHECK-ZNVER4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ZNVER4-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ZNVER4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-ZNVER4-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP1]], align 4
; CHECK-ZNVER4-NEXT:    [[TMP2:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-ZNVER4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[CONDITIONAL_IV]]
; CHECK-ZNVER4-NEXT:    call void @llvm.masked.compressstore.v16i32.p0(<16 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP3]], <16 x i1> [[TMP2]])
; CHECK-ZNVER4-NEXT:    [[TMP4:%.*]] = zext <16 x i1> [[TMP2]] to <16 x i64>
; CHECK-ZNVER4-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[TMP4]])
; CHECK-ZNVER4-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[CONDITIONAL_IV]], [[TMP5]]
; CHECK-ZNVER4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-ZNVER4-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-ZNVER4-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-ZNVER4:       [[MIDDLE_BLOCK]]:
; CHECK-ZNVER4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-ZNVER4-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-ZNVER4:       [[SCALAR_PH]]:
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %idx
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %n
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

define void @expand_load(ptr noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
; CHECK-V4-LABEL: define void @expand_load(
; CHECK-V4-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-V4-NEXT:  [[ENTRY:.*:]]
; CHECK-V4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-V4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-V4:       [[VECTOR_PH]]:
; CHECK-V4-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 7
; CHECK-V4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-V4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[C]], i64 0
; CHECK-V4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
; CHECK-V4-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-V4:       [[VECTOR_BODY]]:
; CHECK-V4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-V4-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-V4-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
; CHECK-V4-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4
; CHECK-V4-NEXT:    [[TMP2:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-V4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[CONDITIONAL_IV]]
; CHECK-V4-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.masked.expandload.v8i32.p0(ptr align 4 [[TMP3]], <8 x i1> [[TMP2]], <8 x i32> poison)
; CHECK-V4-NEXT:    call void @llvm.masked.store.v8i32.p0(<8 x i32> [[TMP4]], ptr align 4 [[TMP1]], <8 x i1> [[TMP2]])
; CHECK-V4-NEXT:    [[TMP5:%.*]] = zext <8 x i1> [[TMP2]] to <8 x i64>
; CHECK-V4-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP5]])
; CHECK-V4-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[CONDITIONAL_IV]], [[TMP6]]
; CHECK-V4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-V4-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-V4-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK-V4:       [[MIDDLE_BLOCK]]:
; CHECK-V4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-V4-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-V4:       [[SCALAR_PH]]:
;
; CHECK-ICELAKE-LABEL: define void @expand_load(
; CHECK-ICELAKE-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-ICELAKE-NEXT:  [[ENTRY:.*:]]
; CHECK-ICELAKE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
; CHECK-ICELAKE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-ICELAKE:       [[VECTOR_PH]]:
; CHECK-ICELAKE-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 7
; CHECK-ICELAKE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-ICELAKE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[C]], i64 0
; CHECK-ICELAKE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
; CHECK-ICELAKE-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-ICELAKE:       [[VECTOR_BODY]]:
; CHECK-ICELAKE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ICELAKE-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ICELAKE-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
; CHECK-ICELAKE-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4
; CHECK-ICELAKE-NEXT:    [[TMP2:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-ICELAKE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[CONDITIONAL_IV]]
; CHECK-ICELAKE-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.masked.expandload.v8i32.p0(ptr align 4 [[TMP3]], <8 x i1> [[TMP2]], <8 x i32> poison)
; CHECK-ICELAKE-NEXT:    call void @llvm.masked.store.v8i32.p0(<8 x i32> [[TMP4]], ptr align 4 [[TMP1]], <8 x i1> [[TMP2]])
; CHECK-ICELAKE-NEXT:    [[TMP5:%.*]] = zext <8 x i1> [[TMP2]] to <8 x i64>
; CHECK-ICELAKE-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP5]])
; CHECK-ICELAKE-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[CONDITIONAL_IV]], [[TMP6]]
; CHECK-ICELAKE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-ICELAKE-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-ICELAKE-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK-ICELAKE:       [[MIDDLE_BLOCK]]:
; CHECK-ICELAKE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-ICELAKE-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-ICELAKE:       [[SCALAR_PH]]:
;
; CHECK-ZNVER4-LABEL: define void @expand_load(
; CHECK-ZNVER4-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-ZNVER4-NEXT:  [[ITER_CHECK:.*]]:
; CHECK-ZNVER4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-ZNVER4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK-ZNVER4:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-ZNVER4-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 16
; CHECK-ZNVER4-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-ZNVER4:       [[VECTOR_PH]]:
; CHECK-ZNVER4-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 15
; CHECK-ZNVER4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-ZNVER4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[C]], i64 0
; CHECK-ZNVER4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
; CHECK-ZNVER4-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-ZNVER4:       [[VECTOR_BODY]]:
; CHECK-ZNVER4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ZNVER4-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ZNVER4-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
; CHECK-ZNVER4-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP1]], align 4
; CHECK-ZNVER4-NEXT:    [[TMP2:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-ZNVER4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[CONDITIONAL_IV]]
; CHECK-ZNVER4-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.masked.expandload.v16i32.p0(ptr align 4 [[TMP3]], <16 x i1> [[TMP2]], <16 x i32> poison)
; CHECK-ZNVER4-NEXT:    call void @llvm.masked.store.v16i32.p0(<16 x i32> [[TMP4]], ptr align 4 [[TMP1]], <16 x i1> [[TMP2]])
; CHECK-ZNVER4-NEXT:    [[TMP5:%.*]] = zext <16 x i1> [[TMP2]] to <16 x i64>
; CHECK-ZNVER4-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[TMP5]])
; CHECK-ZNVER4-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[CONDITIONAL_IV]], [[TMP6]]
; CHECK-ZNVER4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-ZNVER4-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-ZNVER4-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK-ZNVER4:       [[MIDDLE_BLOCK]]:
; CHECK-ZNVER4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-ZNVER4-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK-ZNVER4:       [[VEC_EPILOG_ITER_CHECK]]:
; CHECK-ZNVER4-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; CHECK-ZNVER4-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]]
; CHECK-ZNVER4:       [[VEC_EPILOG_PH]]:
; CHECK-ZNVER4-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-ZNVER4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[CONDITIONAL_STEP]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-ZNVER4-NEXT:    [[TMP8:%.*]] = and i64 [[N]], 3
; CHECK-ZNVER4-NEXT:    [[N_VEC2:%.*]] = sub i64 [[N]], [[TMP8]]
; CHECK-ZNVER4-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-ZNVER4-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-ZNVER4-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; CHECK-ZNVER4:       [[VEC_EPILOG_VECTOR_BODY]]:
; CHECK-ZNVER4-NEXT:    [[INDEX5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-ZNVER4-NEXT:    [[CONDITIONAL_IV6:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[CONDITIONAL_STEP8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-ZNVER4-NEXT:    [[TMP9:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX5]]
; CHECK-ZNVER4-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4
; CHECK-ZNVER4-NEXT:    [[TMP10:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD7]], [[BROADCAST_SPLAT4]]
; CHECK-ZNVER4-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[CONDITIONAL_IV6]]
; CHECK-ZNVER4-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr align 4 [[TMP11]], <4 x i1> [[TMP10]], <4 x i32> poison)
; CHECK-ZNVER4-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP12]], ptr align 4 [[TMP9]], <4 x i1> [[TMP10]])
; CHECK-ZNVER4-NEXT:    [[TMP13:%.*]] = zext <4 x i1> [[TMP10]] to <4 x i64>
; CHECK-ZNVER4-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP13]])
; CHECK-ZNVER4-NEXT:    [[CONDITIONAL_STEP8]] = add i64 [[CONDITIONAL_IV6]], [[TMP14]]
; CHECK-ZNVER4-NEXT:    [[INDEX_NEXT9]] = add nuw i64 [[INDEX5]], 4
; CHECK-ZNVER4-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC2]]
; CHECK-ZNVER4-NEXT:    br i1 [[TMP15]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK-ZNVER4:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-ZNVER4-NEXT:    [[CMP_N10:%.*]] = icmp eq i64 [[N]], [[N_VEC2]]
; CHECK-ZNVER4-NEXT:    br i1 [[CMP_N10]], [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK-ZNVER4:       [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-ZNVER4-NEXT:    [[BC_RESUME_VAL11:%.*]] = phi i64 [ [[N_VEC2]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-ZNVER4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[CONDITIONAL_STEP8]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[CONDITIONAL_STEP]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-ZNVER4-NEXT:    br label %[[FOR_INC:.*]]
; CHECK-ZNVER4:       [[FOR_INC]]:
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv
  %load.dst = load i32, ptr %dst.ptr, align 4
  %cmp = icmp slt i32 %load.dst, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %idx
  %load.src = load i32, ptr %src.ptr, align 4
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %n
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}