llvm/test/Transforms/LoopVectorize/compress-idioms.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
; RUN: opt < %s -force-target-supports-masked-memory-ops -force-vector-width=4 -passes=loop-vectorize -S | FileCheck %s -check-prefixes=CHECK,CHECK-IC1
; RUN: opt < %s -force-target-supports-masked-memory-ops -force-vector-width=4 -tail-folding-policy=must-fold-tail -passes=loop-vectorize -S | FileCheck %s -check-prefixes=CHECK,CHECK-TF

define void @test_compress_store_with_index(ptr writeonly noalias %dst, ptr readonly %src, i32 %c) {
; CHECK-LABEL: define void @test_compress_store_with_index(
; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[CONDITIONAL_IV]]
; CHECK-NEXT:    call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP2]], <4 x i1> [[TMP1]])
; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i64>
; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP3]])
; CHECK-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[CONDITIONAL_IV]], [[TMP4]]
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %idx
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

define void @test_expand_load_with_index(ptr noalias %dst, ptr readonly %src, i32 %c) {
; CHECK-LABEL: define void @test_expand_load_with_index(
; CHECK-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i32, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[CONDITIONAL_IV]]
; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.masked.expandload.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP3]], ptr align 4 [[TMP0]], <4 x i1> [[TMP1]])
; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i64>
; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP4]])
; CHECK-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[CONDITIONAL_IV]], [[TMP5]]
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv
  %load.dst = load i32, ptr %dst.ptr, align 4
  %cmp = icmp slt i32 %load.dst, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %idx
  %load.src = load i32, ptr %src.ptr, align 4
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

define i64 @test_conditionally_incremented_phi_liveout(ptr writeonly noalias %dst, ptr readonly %src, i32 %c) {
; CHECK-LABEL: define i64 @test_conditionally_incremented_phi_liveout(
; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[CONDITIONAL_IV]]
; CHECK-NEXT:    call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP2]], <4 x i1> [[TMP1]])
; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i64>
; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP3]])
; CHECK-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[CONDITIONAL_IV]], [[TMP4]]
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret i64 [[CONDITIONAL_STEP]]
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %idx
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret i64 %idx.1
}

define void @test_compress_store_with_scaled_pointer(ptr writeonly noalias %dst.bytes, ptr readonly %src, i32 %c) {
; CHECK-LABEL: define void @test_compress_store_with_scaled_pointer(
; CHECK-SAME: ptr noalias writeonly [[DST_BYTES:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i64 [[CONDITIONAL_IV]], 2
; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST_BYTES]], i64 [[TMP2]]
; CHECK-NEXT:    call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP3]], <4 x i1> [[TMP1]])
; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i64>
; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP4]])
; CHECK-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[CONDITIONAL_IV]], [[TMP5]]
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %idx.bytes = shl nsw i64 %idx, 2
  %dst.ptr = getelementptr inbounds i8, ptr %dst.bytes, i64 %idx.bytes
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

; Test a nested conditional compress store, where the phi is only updated on iterations where the store takes place.
define void @test_nested_conditional_compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %c) {
; CHECK-LABEL: define void @test_nested_conditional_compress_store(
; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[CONDITIONAL_IV]]
; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
; CHECK-NEXT:    call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP2]], <4 x i1> [[TMP4]])
; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i1> [[TMP4]] to <4 x i64>
; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP5]])
; CHECK-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[CONDITIONAL_IV]], [[TMP6]]
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
  %idx = phi i64 [ 0, %entry ], [ %phi, %latch ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br i1 %cmp, label %update.block, label %latch

update.block:
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %idx
  %update = add nsw i64 %idx, 1
  %cmp2 = icmp sgt i32 %load.src, 0
  br i1 %cmp2, label %store.block, label %latch

store.block:
  store i32 %load.src, ptr %dst.ptr, align 4
  br label %latch

latch:
  %phi = phi i64 [ %idx, %for.body ], [ %idx, %update.block ], [ %update, %store.block ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

; An unconditional increment should lower as a simple induction (not a conditional induction).
define void @test_unconditional_increment(ptr writeonly noalias %dst, ptr readonly %src, i32 %c) {
; CHECK-LABEL: define void @test_unconditional_increment(
; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = add i64 15, [[INDEX]]
; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP0]]
; CHECK-NEXT:    store <4 x i32> [[WIDE_LOAD]], ptr [[TMP2]], align 4
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
; CHECK-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 15, %entry ], [ %idx.1, %for.inc ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br label %inc.step

inc.step:
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %idx
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %inc.step ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

define void @test_multiple_conditional_inductions(ptr %dst, ptr noalias %dst2, ptr noalias %src, ptr noalias %cond, ptr noalias %cond2) {
; CHECK-LABEL: define void @test_multiple_conditional_inductions(
; CHECK-SAME: ptr [[DST:%.*]], ptr noalias [[DST2:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[COND:%.*]], ptr noalias [[COND2:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[CONDITIONAL_IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[CONDITIONAL_IV]]
; CHECK-NEXT:    [[TMP4:%.*]] = trunc <4 x i32> [[WIDE_LOAD2]] to <4 x i8>
; CHECK-NEXT:    call void @llvm.masked.compressstore.v4i8.p0(<4 x i8> [[TMP4]], ptr align 1 [[TMP3]], <4 x i1> [[TMP1]])
; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i64>
; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP5]])
; CHECK-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[CONDITIONAL_IV]], [[TMP6]]
; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[COND2]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP7]], align 4
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD3]], zeroinitializer
; CHECK-NEXT:    [[TMP9:%.*]] = trunc <4 x i32> [[WIDE_LOAD2]] to <4 x i16>
; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[DST2]], i64 [[CONDITIONAL_IV1]]
; CHECK-NEXT:    call void @llvm.masked.compressstore.v4i16.p0(<4 x i16> [[TMP9]], ptr align 2 [[TMP10]], <4 x i1> [[TMP8]])
; CHECK-NEXT:    [[TMP11:%.*]] = zext <4 x i1> [[TMP8]] to <4 x i64>
; CHECK-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP11]])
; CHECK-NEXT:    [[CONDITIONAL_STEP4]] = add i64 [[CONDITIONAL_IV1]], [[TMP12]]
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
; CHECK-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %dst.idx = phi i64 [ 0, %entry ], [ %dst.inc, %for.inc ]
  %dst2.idx = phi i64 [ 0, %entry ], [ %dst2.inc, %for.inc ]
  %cond.gep = getelementptr inbounds i32, ptr %cond, i64 %iv
  %cond.val = load i32, ptr %cond.gep, align 4
  %cond.is.zero = icmp eq i32 %cond.val, 0
  %src.gep = getelementptr inbounds i32, ptr %src, i64 %iv
  %src.val = load i32, ptr %src.gep, align 4
  br i1 %cond.is.zero, label %if.end, label %if.then0

if.then0:
  %dst.idx.next = add nsw i64 %dst.idx, 1
  %dst.gep = getelementptr inbounds i8, ptr %dst, i64 %dst.idx
  %dst.val.trunc = trunc i32 %src.val to i8
  store i8 %dst.val.trunc, ptr %dst.gep, align 1
  br label %if.end

if.end:
  %dst.inc = phi i64 [ %dst.idx.next, %if.then0 ], [ %dst.idx, %for.body ]
  %cond2.gep = getelementptr inbounds i32, ptr %cond2, i64 %iv
  %cond2.val = load i32, ptr %cond2.gep, align 4
  %cond2.is.zero = icmp eq i32 %cond2.val, 0
  br i1 %cond2.is.zero, label %for.inc, label %if.then1

if.then1:
  %dst2.val.trunc = trunc i32 %src.val to i16
  %dst2.idx.next = add nsw i64 %dst2.idx, 1
  %dst2.gep = getelementptr inbounds i16, ptr %dst2, i64 %dst2.idx
  store i16 %dst2.val.trunc, ptr %dst2.gep, align 2
  br label %for.inc

for.inc:
  %dst2.inc = phi i64 [ %dst2.idx.next, %if.then1 ], [ %dst2.idx, %if.end ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

; Test expand load with always false update (this probably can be simplified).
define void @test_expand_load_always_false_cond(ptr noalias %dst, ptr readonly %src, i32 %c) {
; CHECK-LABEL: define void @test_expand_load_always_false_cond(
; CHECK-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
; CHECK-NEXT:    [[TMP6:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[CONDITIONAL_IV]]
; CHECK-NEXT:    call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP7]], <4 x i1> [[TMP6]])
; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i1> [[TMP6]] to <4 x i64>
; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP3]])
; CHECK-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[CONDITIONAL_IV]], [[TMP4]]
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 4
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %idx
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

define void @test_compress_with_unknown_tc(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
; CHECK-IC1-LABEL: define void @test_compress_with_unknown_tc(
; CHECK-IC1-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
; CHECK-IC1-NEXT:  [[ENTRY:.*:]]
; CHECK-IC1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-IC1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-IC1:       [[VECTOR_PH]]:
; CHECK-IC1-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-IC1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-IC1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-IC1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-IC1:       [[VECTOR_BODY]]:
; CHECK-IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
; CHECK-IC1-NEXT:    [[TMP2:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-IC1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[CONDITIONAL_IV]]
; CHECK-IC1-NEXT:    call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[TMP3]], <4 x i1> [[TMP2]])
; CHECK-IC1-NEXT:    [[TMP4:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i64>
; CHECK-IC1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP4]])
; CHECK-IC1-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[CONDITIONAL_IV]], [[TMP5]]
; CHECK-IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-IC1-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-IC1-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-IC1:       [[MIDDLE_BLOCK]]:
; CHECK-IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-IC1-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-IC1:       [[SCALAR_PH]]:
;
; CHECK-TF-LABEL: define void @test_compress_with_unknown_tc(
; CHECK-TF-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]], i64 [[N:%.*]]) {
; CHECK-TF-NEXT:  [[ENTRY:.*:]]
; CHECK-TF-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK-TF:       [[VECTOR_PH]]:
; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], 3
; CHECK-TF-NEXT:    [[TMP0:%.*]] = and i64 [[N_RND_UP]], 3
; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP0]]
; CHECK-TF-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
; CHECK-TF-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-TF-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-TF-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-TF-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-TF:       [[VECTOR_BODY]]:
; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-TF-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp slt <4 x i32> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT2]]
; CHECK-TF-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
; CHECK-TF-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[CONDITIONAL_IV]]
; CHECK-TF-NEXT:    call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP5]], <4 x i1> [[TMP4]])
; CHECK-TF-NEXT:    [[TMP6:%.*]] = zext <4 x i1> [[TMP4]] to <4 x i64>
; CHECK-TF-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP6]])
; CHECK-TF-NEXT:    [[CONDITIONAL_STEP]] = add i64 [[CONDITIONAL_IV]], [[TMP7]]
; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; CHECK-TF-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-TF-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-TF-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-TF:       [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT:    br label %[[EXIT:.*]]
; CHECK-TF:       [[EXIT]]:
; CHECK-TF-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %idx
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %n
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

; Test a single iteration loop where the backedge update is removed from the conditional induction.
define void @compress_store_single_iteration(ptr writeonly noalias %dst, ptr readonly %src, i32 %c) {
; CHECK-LABEL: define void @compress_store_single_iteration(
; CHECK-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[SRC]], align 4
; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> [[WIDE_LOAD]], ptr align 4 [[DST]], <4 x i1> [[TMP0]])
; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %idx
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

; Test vectorizing a conditional ptr induction without any compressed memops.
define void @conditional_pointer_induction_with_no_compressed_memops(ptr %start, ptr %cond) {
; CHECK-IC1-LABEL: define void @conditional_pointer_induction_with_no_compressed_memops(
; CHECK-IC1-SAME: ptr [[START:%.*]], ptr [[COND:%.*]]) {
; CHECK-IC1-NEXT:  [[ENTRY:.*:]]
; CHECK-IC1-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK-IC1:       [[VECTOR_PH]]:
; CHECK-IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-IC1:       [[VECTOR_BODY]]:
; CHECK-IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-IC1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-IC1-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK-IC1:       [[MIDDLE_BLOCK]]:
; CHECK-IC1-NEXT:    br label %[[EXIT:.*]]
; CHECK-IC1:       [[EXIT]]:
; CHECK-IC1-NEXT:    ret void
;
; CHECK-TF-LABEL: define void @conditional_pointer_induction_with_no_compressed_memops(
; CHECK-TF-SAME: ptr [[START:%.*]], ptr [[COND:%.*]]) {
; CHECK-TF-NEXT:  [[ENTRY:.*:]]
; CHECK-TF-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK-TF:       [[VECTOR_PH]]:
; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-TF:       [[VECTOR_BODY]]:
; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-TF-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-TF-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK-TF:       [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT:    br label %[[EXIT:.*]]
; CHECK-TF:       [[EXIT]]:
; CHECK-TF-NEXT:    ret void
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
  %conditional.ptr = phi ptr [ %start, %entry ], [ %conditional.ptr.next, %latch ]
  %cond.ptr = getelementptr inbounds i32, ptr %cond, i64 %iv
  %cond.val = load i32, ptr %cond.ptr, align 4
  %cmp = icmp eq i32 %cond.val, 1
  br i1 %cmp, label %if.then, label %latch

if.then:
  %conditional.step = getelementptr i8, ptr %conditional.ptr, i64 40
  br label %latch

latch:
  %conditional.ptr.next = phi ptr [ %conditional.step, %if.then ], [ %conditional.ptr, %loop ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond = icmp eq i64 %iv.next, 1024
  br i1 %exitcond, label %exit, label %loop

exit:
  ret void
}

; Regression test to make sure we don't insert NumActiveLanes(mask) before the def of "mask".
; TODO: We should avoid the broadcast for the tail folded liveout.
define ptr @conditional_ptr_phi_with_liveout(ptr %start, ptr %c, i64 %n) {
; CHECK-IC1-LABEL: define ptr @conditional_ptr_phi_with_liveout(
; CHECK-IC1-SAME: ptr [[START:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) {
; CHECK-IC1-NEXT:  [[ENTRY:.*:]]
; CHECK-IC1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-IC1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-IC1:       [[VECTOR_PH]]:
; CHECK-IC1-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-IC1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-IC1:       [[VECTOR_BODY]]:
; CHECK-IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT:    [[CONDITIONAL_IV:%.*]] = phi ptr [ [[START]], %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
; CHECK-IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
; CHECK-IC1-NEXT:    [[TMP2:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], zeroinitializer
; CHECK-IC1-NEXT:    [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i64>
; CHECK-IC1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP3]])
; CHECK-IC1-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 2
; CHECK-IC1-NEXT:    [[CONDITIONAL_STEP]] = getelementptr i8, ptr [[CONDITIONAL_IV]], i64 [[TMP5]]
; CHECK-IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-IC1-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-IC1-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK-IC1:       [[MIDDLE_BLOCK]]:
; CHECK-IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-IC1-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-IC1:       [[SCALAR_PH]]:
;
; CHECK-TF-LABEL: define ptr @conditional_ptr_phi_with_liveout(
; CHECK-TF-SAME: ptr [[START:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) {
; CHECK-TF-NEXT:  [[ENTRY:.*:]]
; CHECK-TF-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK-TF:       [[VECTOR_PH]]:
; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], 3
; CHECK-TF-NEXT:    [[TMP0:%.*]] = and i64 [[N_RND_UP]], 3
; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP0]]
; CHECK-TF-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
; CHECK-TF-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-TF-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-TF:       [[VECTOR_BODY]]:
; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[CONDITIONAL_IV:%.*]] = phi ptr [ [[START]], %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-TF-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP2]], <4 x i1> [[TMP1]], <4 x i8> poison)
; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ne <4 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
; CHECK-TF-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
; CHECK-TF-NEXT:    [[TMP5:%.*]] = zext <4 x i1> [[TMP4]] to <4 x i64>
; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP5]])
; CHECK-TF-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 2
; CHECK-TF-NEXT:    [[CONDITIONAL_STEP]] = getelementptr i8, ptr [[CONDITIONAL_IV]], i64 [[TMP7]]
; CHECK-TF-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x ptr> poison, ptr [[CONDITIONAL_STEP]], i64 0
; CHECK-TF-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT1]], <4 x ptr> poison, <4 x i32> zeroinitializer
; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; CHECK-TF-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-TF-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-TF-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK-TF:       [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT:    [[TMP9:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
; CHECK-TF-NEXT:    [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP9]], i1 false)
; CHECK-TF-NEXT:    [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
; CHECK-TF-NEXT:    [[TMP10:%.*]] = extractelement <4 x ptr> [[BROADCAST_SPLAT2]], i64 [[LAST_ACTIVE_LANE]]
; CHECK-TF-NEXT:    br label %[[EXIT:.*]]
; CHECK-TF:       [[EXIT]]:
; CHECK-TF-NEXT:    ret ptr [[TMP10]]
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
  %p = phi ptr [ %start, %entry ], [ %p.1, %latch ]
  %gep.c = getelementptr i8, ptr %c, i64 %iv
  %cv = load i8, ptr %gep.c
  %cmp = icmp eq i8 %cv, 0
  %p.next = getelementptr i8, ptr %p, i64 4
  br i1 %cmp, label %skip, label %latch

skip:
  br label %latch

latch:
  %p.1 = phi ptr [ %p.next, %loop ], [ %p, %skip ]
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret ptr %p.1
}

; TODO: We should avoid the broadcast for the tail folded liveout.
define ptr @conditional_ptr_phi_non_const_step_with_liveout(ptr %start, ptr %c, i64 %n, i64 %step) {
; CHECK-IC1-LABEL: define ptr @conditional_ptr_phi_non_const_step_with_liveout(
; CHECK-IC1-SAME: ptr [[START:%.*]], ptr [[C:%.*]], i64 [[N:%.*]], i64 [[STEP:%.*]]) {
; CHECK-IC1-NEXT:  [[ENTRY:.*:]]
; CHECK-IC1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-IC1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-IC1:       [[VECTOR_PH]]:
; CHECK-IC1-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-IC1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-IC1-NEXT:    [[TMP7:%.*]] = freeze i64 [[STEP]]
; CHECK-IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-IC1:       [[VECTOR_BODY]]:
; CHECK-IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT:    [[CONDITIONAL_IV:%.*]] = phi ptr [ [[START]], %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
; CHECK-IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
; CHECK-IC1-NEXT:    [[TMP2:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], zeroinitializer
; CHECK-IC1-NEXT:    [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i64>
; CHECK-IC1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP3]])
; CHECK-IC1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP7]], [[TMP4]]
; CHECK-IC1-NEXT:    [[CONDITIONAL_STEP]] = getelementptr i8, ptr [[CONDITIONAL_IV]], i64 [[TMP5]]
; CHECK-IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-IC1-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-IC1-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK-IC1:       [[MIDDLE_BLOCK]]:
; CHECK-IC1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-IC1-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-IC1:       [[SCALAR_PH]]:
;
; CHECK-TF-LABEL: define ptr @conditional_ptr_phi_non_const_step_with_liveout(
; CHECK-TF-SAME: ptr [[START:%.*]], ptr [[C:%.*]], i64 [[N:%.*]], i64 [[STEP:%.*]]) {
; CHECK-TF-NEXT:  [[ENTRY:.*:]]
; CHECK-TF-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK-TF:       [[VECTOR_PH]]:
; CHECK-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], 3
; CHECK-TF-NEXT:    [[TMP0:%.*]] = and i64 [[N_RND_UP]], 3
; CHECK-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP0]]
; CHECK-TF-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
; CHECK-TF-NEXT:    [[TMP11:%.*]] = freeze i64 [[STEP]]
; CHECK-TF-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-TF-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-TF:       [[VECTOR_BODY]]:
; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[CONDITIONAL_IV:%.*]] = phi ptr [ [[START]], %[[VECTOR_PH]] ], [ [[CONDITIONAL_STEP:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-TF-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
; CHECK-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP2]], <4 x i1> [[TMP1]], <4 x i8> poison)
; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp ne <4 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
; CHECK-TF-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer
; CHECK-TF-NEXT:    [[TMP5:%.*]] = zext <4 x i1> [[TMP4]] to <4 x i64>
; CHECK-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP5]])
; CHECK-TF-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP11]], [[TMP6]]
; CHECK-TF-NEXT:    [[CONDITIONAL_STEP]] = getelementptr i8, ptr [[CONDITIONAL_IV]], i64 [[TMP7]]
; CHECK-TF-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x ptr> poison, ptr [[CONDITIONAL_STEP]], i64 0
; CHECK-TF-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT1]], <4 x ptr> poison, <4 x i32> zeroinitializer
; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; CHECK-TF-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-TF-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-TF-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK-TF:       [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT:    [[TMP9:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
; CHECK-TF-NEXT:    [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP9]], i1 false)
; CHECK-TF-NEXT:    [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
; CHECK-TF-NEXT:    [[TMP10:%.*]] = extractelement <4 x ptr> [[BROADCAST_SPLAT2]], i64 [[LAST_ACTIVE_LANE]]
; CHECK-TF-NEXT:    br label %[[EXIT:.*]]
; CHECK-TF:       [[EXIT]]:
; CHECK-TF-NEXT:    ret ptr [[TMP10]]
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
  %p = phi ptr [ %start, %entry ], [ %p.1, %latch ]
  %gep.c = getelementptr i8, ptr %c, i64 %iv
  %cv = load i8, ptr %gep.c
  %cmp = icmp eq i8 %cv, 0
  %p.next = getelementptr i8, ptr %p, i64 %step
  br i1 %cmp, label %skip, label %latch

skip:
  br label %latch

latch:
  %p.1 = phi ptr [ %p.next, %loop ], [ %p, %skip ]
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret ptr %p.1
}