llvm/test/Transforms/LoopVectorize/compress-idioms.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt < %s -force-target-supports-masked-memory-ops -force-vector-width=4 -passes=loop-vectorize -S | FileCheck %s -check-prefix=CHECK-IC1
; RUN: opt < %s -force-target-supports-masked-memory-ops -force-vector-width=4 -tail-folding-policy=must-fold-tail -passes=loop-vectorize -S | FileCheck %s -check-prefix=CHECK-TF

define void @test_compress_store_with_index(ptr writeonly noalias %dst, ptr readonly %src, i32 %c) {
; CHECK-IC1-LABEL: define void @test_compress_store_with_index(
; CHECK-IC1-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-IC1-NEXT:  [[VECTOR_PH:.*]]:
; CHECK-IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-IC1:       [[VECTOR_BODY]]:
; CHECK-IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[EXIT:.*]] ]
; CHECK-IC1-NEXT:    [[IDX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IDX_1:%.*]], %[[EXIT]] ]
; CHECK-IC1-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-IC1-NEXT:    [[LOAD_SRC:%.*]] = load i32, ptr [[TMP0]], align 4
; CHECK-IC1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[LOAD_SRC]], [[C]]
; CHECK-IC1-NEXT:    br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]]
; CHECK-IC1:       [[MIDDLE_BLOCK]]:
; CHECK-IC1-NEXT:    [[DST_PTR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IDX]]
; CHECK-IC1-NEXT:    store i32 [[LOAD_SRC]], ptr [[DST_PTR]], align 4
; CHECK-IC1-NEXT:    [[IDX_NEXT:%.*]] = add nsw i64 [[IDX]], 1
; CHECK-IC1-NEXT:    br label %[[EXIT]]
; CHECK-IC1:       [[EXIT]]:
; CHECK-IC1-NEXT:    [[IDX_1]] = phi i64 [ [[IDX_NEXT]], %[[MIDDLE_BLOCK]] ], [ [[IDX]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[INDEX]], 1
; CHECK-IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
; CHECK-IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]]
; CHECK-IC1:       [[EXIT1]]:
; CHECK-IC1-NEXT:    ret void
;
; CHECK-TF-LABEL: define void @test_compress_store_with_index(
; CHECK-TF-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-TF-NEXT:  [[VECTOR_PH:.*]]:
; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-TF:       [[VECTOR_BODY]]:
; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[EXIT:.*]] ]
; CHECK-TF-NEXT:    [[IDX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IDX_1:%.*]], %[[EXIT]] ]
; CHECK-TF-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-TF-NEXT:    [[LOAD_SRC:%.*]] = load i32, ptr [[TMP0]], align 4
; CHECK-TF-NEXT:    [[CMP:%.*]] = icmp slt i32 [[LOAD_SRC]], [[C]]
; CHECK-TF-NEXT:    br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]]
; CHECK-TF:       [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT:    [[DST_PTR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IDX]]
; CHECK-TF-NEXT:    store i32 [[LOAD_SRC]], ptr [[DST_PTR]], align 4
; CHECK-TF-NEXT:    [[IDX_NEXT:%.*]] = add nsw i64 [[IDX]], 1
; CHECK-TF-NEXT:    br label %[[EXIT]]
; CHECK-TF:       [[EXIT]]:
; CHECK-TF-NEXT:    [[IDX_1]] = phi i64 [ [[IDX_NEXT]], %[[MIDDLE_BLOCK]] ], [ [[IDX]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[INDEX]], 1
; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]]
; CHECK-TF:       [[EXIT1]]:
; CHECK-TF-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %idx
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

define void @test_expand_load_with_index(ptr noalias %dst, ptr readonly %src, i32 %c) {
; CHECK-IC1-LABEL: define void @test_expand_load_with_index(
; CHECK-IC1-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-IC1-NEXT:  [[ENTRY:.*]]:
; CHECK-IC1-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK-IC1:       [[VECTOR_PH]]:
; CHECK-IC1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[EXIT:.*]] ]
; CHECK-IC1-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_1:%.*]], %[[EXIT]] ]
; CHECK-IC1-NEXT:    [[DST_PTR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
; CHECK-IC1-NEXT:    [[LOAD_DST:%.*]] = load i32, ptr [[DST_PTR]], align 4
; CHECK-IC1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[LOAD_DST]], [[C]]
; CHECK-IC1-NEXT:    br i1 [[CMP]], label %[[VECTOR_BODY:.*]], label %[[EXIT]]
; CHECK-IC1:       [[VECTOR_BODY]]:
; CHECK-IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[CONDITIONAL_IV]]
; CHECK-IC1-NEXT:    [[LOAD_SRC:%.*]] = load i32, ptr [[TMP2]], align 4
; CHECK-IC1-NEXT:    store i32 [[LOAD_SRC]], ptr [[DST_PTR]], align 4
; CHECK-IC1-NEXT:    [[IDX_NEXT:%.*]] = add nsw i64 [[CONDITIONAL_IV]], 1
; CHECK-IC1-NEXT:    br label %[[EXIT]]
; CHECK-IC1:       [[EXIT]]:
; CHECK-IC1-NEXT:    [[IDX_1]] = phi i64 [ [[IDX_NEXT]], %[[VECTOR_BODY]] ], [ [[CONDITIONAL_IV]], %[[VECTOR_PH]] ]
; CHECK-IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
; CHECK-IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_PH]]
; CHECK-IC1:       [[EXIT1]]:
; CHECK-IC1-NEXT:    ret void
;
; CHECK-TF-LABEL: define void @test_expand_load_with_index(
; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-TF-NEXT:  [[ENTRY:.*]]:
; CHECK-TF-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK-TF:       [[VECTOR_PH]]:
; CHECK-TF-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[EXIT:.*]] ]
; CHECK-TF-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_1:%.*]], %[[EXIT]] ]
; CHECK-TF-NEXT:    [[DST_PTR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
; CHECK-TF-NEXT:    [[LOAD_DST:%.*]] = load i32, ptr [[DST_PTR]], align 4
; CHECK-TF-NEXT:    [[CMP:%.*]] = icmp slt i32 [[LOAD_DST]], [[C]]
; CHECK-TF-NEXT:    br i1 [[CMP]], label %[[VECTOR_BODY:.*]], label %[[EXIT]]
; CHECK-TF:       [[VECTOR_BODY]]:
; CHECK-TF-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[CONDITIONAL_IV]]
; CHECK-TF-NEXT:    [[LOAD_SRC:%.*]] = load i32, ptr [[TMP2]], align 4
; CHECK-TF-NEXT:    store i32 [[LOAD_SRC]], ptr [[DST_PTR]], align 4
; CHECK-TF-NEXT:    [[IDX_NEXT:%.*]] = add nsw i64 [[CONDITIONAL_IV]], 1
; CHECK-TF-NEXT:    br label %[[EXIT]]
; CHECK-TF:       [[EXIT]]:
; CHECK-TF-NEXT:    [[IDX_1]] = phi i64 [ [[IDX_NEXT]], %[[VECTOR_BODY]] ], [ [[CONDITIONAL_IV]], %[[VECTOR_PH]] ]
; CHECK-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_PH]]
; CHECK-TF:       [[EXIT1]]:
; CHECK-TF-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv
  %load.dst = load i32, ptr %dst.ptr, align 4
  %cmp = icmp slt i32 %load.dst, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %idx
  %load.src = load i32, ptr %src.ptr, align 4
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

define i64 @test_conditionally_incremented_phi_liveout(ptr writeonly noalias %dst, ptr readonly %src, i32 %c) {
; CHECK-IC1-LABEL: define i64 @test_conditionally_incremented_phi_liveout(
; CHECK-IC1-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-IC1-NEXT:  [[VECTOR_PH:.*]]:
; CHECK-IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-IC1:       [[VECTOR_BODY]]:
; CHECK-IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[EXIT:.*]] ]
; CHECK-IC1-NEXT:    [[IDX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IDX_1:%.*]], %[[EXIT]] ]
; CHECK-IC1-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-IC1-NEXT:    [[LOAD_SRC:%.*]] = load i32, ptr [[TMP0]], align 4
; CHECK-IC1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[LOAD_SRC]], [[C]]
; CHECK-IC1-NEXT:    br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]]
; CHECK-IC1:       [[MIDDLE_BLOCK]]:
; CHECK-IC1-NEXT:    [[DST_PTR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IDX]]
; CHECK-IC1-NEXT:    store i32 [[LOAD_SRC]], ptr [[DST_PTR]], align 4
; CHECK-IC1-NEXT:    [[IDX_NEXT:%.*]] = add nsw i64 [[IDX]], 1
; CHECK-IC1-NEXT:    br label %[[EXIT]]
; CHECK-IC1:       [[EXIT]]:
; CHECK-IC1-NEXT:    [[IDX_1]] = phi i64 [ [[IDX_NEXT]], %[[MIDDLE_BLOCK]] ], [ [[IDX]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[INDEX]], 1
; CHECK-IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
; CHECK-IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]]
; CHECK-IC1:       [[EXIT1]]:
; CHECK-IC1-NEXT:    [[CONDITIONAL_STEP:%.*]] = phi i64 [ [[IDX_1]], %[[EXIT]] ]
; CHECK-IC1-NEXT:    ret i64 [[CONDITIONAL_STEP]]
;
; CHECK-TF-LABEL: define i64 @test_conditionally_incremented_phi_liveout(
; CHECK-TF-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-TF-NEXT:  [[VECTOR_PH:.*]]:
; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-TF:       [[VECTOR_BODY]]:
; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[EXIT:.*]] ]
; CHECK-TF-NEXT:    [[IDX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IDX_1:%.*]], %[[EXIT]] ]
; CHECK-TF-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-TF-NEXT:    [[LOAD_SRC:%.*]] = load i32, ptr [[TMP0]], align 4
; CHECK-TF-NEXT:    [[CMP:%.*]] = icmp slt i32 [[LOAD_SRC]], [[C]]
; CHECK-TF-NEXT:    br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]]
; CHECK-TF:       [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT:    [[DST_PTR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IDX]]
; CHECK-TF-NEXT:    store i32 [[LOAD_SRC]], ptr [[DST_PTR]], align 4
; CHECK-TF-NEXT:    [[IDX_NEXT:%.*]] = add nsw i64 [[IDX]], 1
; CHECK-TF-NEXT:    br label %[[EXIT]]
; CHECK-TF:       [[EXIT]]:
; CHECK-TF-NEXT:    [[IDX_1]] = phi i64 [ [[IDX_NEXT]], %[[MIDDLE_BLOCK]] ], [ [[IDX]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[INDEX]], 1
; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]]
; CHECK-TF:       [[EXIT1]]:
; CHECK-TF-NEXT:    [[CONDITIONAL_STEP:%.*]] = phi i64 [ [[IDX_1]], %[[EXIT]] ]
; CHECK-TF-NEXT:    ret i64 [[CONDITIONAL_STEP]]
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %idx
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret i64 %idx.1
}

define void @test_compress_store_with_scaled_pointer(ptr writeonly noalias %dst.bytes, ptr readonly %src, i32 %c) {
; CHECK-IC1-LABEL: define void @test_compress_store_with_scaled_pointer(
; CHECK-IC1-SAME: ptr noalias writeonly [[DST_BYTES:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-IC1-NEXT:  [[VECTOR_PH:.*]]:
; CHECK-IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-IC1:       [[VECTOR_BODY]]:
; CHECK-IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[EXIT:.*]] ]
; CHECK-IC1-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IDX_1:%.*]], %[[EXIT]] ]
; CHECK-IC1-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-IC1-NEXT:    [[LOAD_SRC:%.*]] = load i32, ptr [[TMP0]], align 4
; CHECK-IC1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[LOAD_SRC]], [[C]]
; CHECK-IC1-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[EXIT]]
; CHECK-IC1:       [[IF_THEN]]:
; CHECK-IC1-NEXT:    [[TMP2:%.*]] = shl nsw i64 [[CONDITIONAL_IV]], 2
; CHECK-IC1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST_BYTES]], i64 [[TMP2]]
; CHECK-IC1-NEXT:    store i32 [[LOAD_SRC]], ptr [[TMP3]], align 4
; CHECK-IC1-NEXT:    [[IDX_NEXT:%.*]] = add nsw i64 [[CONDITIONAL_IV]], 1
; CHECK-IC1-NEXT:    br label %[[EXIT]]
; CHECK-IC1:       [[EXIT]]:
; CHECK-IC1-NEXT:    [[IDX_1]] = phi i64 [ [[IDX_NEXT]], %[[IF_THEN]] ], [ [[CONDITIONAL_IV]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[INDEX]], 1
; CHECK-IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
; CHECK-IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]]
; CHECK-IC1:       [[EXIT1]]:
; CHECK-IC1-NEXT:    ret void
;
; CHECK-TF-LABEL: define void @test_compress_store_with_scaled_pointer(
; CHECK-TF-SAME: ptr noalias writeonly [[DST_BYTES:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-TF-NEXT:  [[VECTOR_PH:.*]]:
; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-TF:       [[VECTOR_BODY]]:
; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[EXIT:.*]] ]
; CHECK-TF-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IDX_1:%.*]], %[[EXIT]] ]
; CHECK-TF-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-TF-NEXT:    [[LOAD_SRC:%.*]] = load i32, ptr [[TMP0]], align 4
; CHECK-TF-NEXT:    [[CMP:%.*]] = icmp slt i32 [[LOAD_SRC]], [[C]]
; CHECK-TF-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[EXIT]]
; CHECK-TF:       [[IF_THEN]]:
; CHECK-TF-NEXT:    [[TMP2:%.*]] = shl nsw i64 [[CONDITIONAL_IV]], 2
; CHECK-TF-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST_BYTES]], i64 [[TMP2]]
; CHECK-TF-NEXT:    store i32 [[LOAD_SRC]], ptr [[TMP3]], align 4
; CHECK-TF-NEXT:    [[IDX_NEXT:%.*]] = add nsw i64 [[CONDITIONAL_IV]], 1
; CHECK-TF-NEXT:    br label %[[EXIT]]
; CHECK-TF:       [[EXIT]]:
; CHECK-TF-NEXT:    [[IDX_1]] = phi i64 [ [[IDX_NEXT]], %[[IF_THEN]] ], [ [[CONDITIONAL_IV]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[INDEX]], 1
; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]]
; CHECK-TF:       [[EXIT1]]:
; CHECK-TF-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %idx.bytes = shl nsw i64 %idx, 2
  %dst.ptr = getelementptr inbounds i8, ptr %dst.bytes, i64 %idx.bytes
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

; Test a nested conditional compress store, where the phi is only updated on iterations where the store takes place.
define void @test_nested_conditional_compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %c) {
; CHECK-IC1-LABEL: define void @test_nested_conditional_compress_store(
; CHECK-IC1-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-IC1-NEXT:  [[VECTOR_PH:.*]]:
; CHECK-IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-IC1:       [[VECTOR_BODY]]:
; CHECK-IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[EXIT:.*]] ]
; CHECK-IC1-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[PHI:%.*]], %[[EXIT]] ]
; CHECK-IC1-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-IC1-NEXT:    [[LOAD_SRC:%.*]] = load i32, ptr [[TMP0]], align 4
; CHECK-IC1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[LOAD_SRC]], [[C]]
; CHECK-IC1-NEXT:    br i1 [[CMP]], label %[[UPDATE_BLOCK:.*]], label %[[EXIT]]
; CHECK-IC1:       [[UPDATE_BLOCK]]:
; CHECK-IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[CONDITIONAL_IV]]
; CHECK-IC1-NEXT:    [[UPDATE:%.*]] = add nsw i64 [[CONDITIONAL_IV]], 1
; CHECK-IC1-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[LOAD_SRC]], 0
; CHECK-IC1-NEXT:    br i1 [[CMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]]
; CHECK-IC1:       [[MIDDLE_BLOCK]]:
; CHECK-IC1-NEXT:    store i32 [[LOAD_SRC]], ptr [[TMP2]], align 4
; CHECK-IC1-NEXT:    br label %[[EXIT]]
; CHECK-IC1:       [[EXIT]]:
; CHECK-IC1-NEXT:    [[PHI]] = phi i64 [ [[CONDITIONAL_IV]], %[[VECTOR_BODY]] ], [ [[CONDITIONAL_IV]], %[[UPDATE_BLOCK]] ], [ [[UPDATE]], %[[MIDDLE_BLOCK]] ]
; CHECK-IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[INDEX]], 1
; CHECK-IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
; CHECK-IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]]
; CHECK-IC1:       [[EXIT1]]:
; CHECK-IC1-NEXT:    ret void
;
; CHECK-TF-LABEL: define void @test_nested_conditional_compress_store(
; CHECK-TF-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-TF-NEXT:  [[VECTOR_PH:.*]]:
; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-TF:       [[VECTOR_BODY]]:
; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[EXIT:.*]] ]
; CHECK-TF-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[PHI:%.*]], %[[EXIT]] ]
; CHECK-TF-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-TF-NEXT:    [[LOAD_SRC:%.*]] = load i32, ptr [[TMP0]], align 4
; CHECK-TF-NEXT:    [[CMP:%.*]] = icmp slt i32 [[LOAD_SRC]], [[C]]
; CHECK-TF-NEXT:    br i1 [[CMP]], label %[[UPDATE_BLOCK:.*]], label %[[EXIT]]
; CHECK-TF:       [[UPDATE_BLOCK]]:
; CHECK-TF-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[CONDITIONAL_IV]]
; CHECK-TF-NEXT:    [[UPDATE:%.*]] = add nsw i64 [[CONDITIONAL_IV]], 1
; CHECK-TF-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[LOAD_SRC]], 0
; CHECK-TF-NEXT:    br i1 [[CMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[EXIT]]
; CHECK-TF:       [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT:    store i32 [[LOAD_SRC]], ptr [[TMP2]], align 4
; CHECK-TF-NEXT:    br label %[[EXIT]]
; CHECK-TF:       [[EXIT]]:
; CHECK-TF-NEXT:    [[PHI]] = phi i64 [ [[CONDITIONAL_IV]], %[[VECTOR_BODY]] ], [ [[CONDITIONAL_IV]], %[[UPDATE_BLOCK]] ], [ [[UPDATE]], %[[MIDDLE_BLOCK]] ]
; CHECK-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[INDEX]], 1
; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]]
; CHECK-TF:       [[EXIT1]]:
; CHECK-TF-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
  %idx = phi i64 [ 0, %entry ], [ %phi, %latch ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br i1 %cmp, label %update.block, label %latch

update.block:
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %idx
  %update = add nsw i64 %idx, 1
  %cmp2 = icmp sgt i32 %load.src, 0
  br i1 %cmp2, label %store.block, label %latch

store.block:
  store i32 %load.src, ptr %dst.ptr, align 4
  br label %latch

latch:
  %phi = phi i64 [ %idx, %for.body ], [ %idx, %update.block ], [ %update, %store.block ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

; An unconditional increment should lower as a simple induction (not a conditional induction).
define void @test_unconditional_increment(ptr writeonly noalias %dst, ptr readonly %src, i32 %c) {
; CHECK-IC1-LABEL: define void @test_unconditional_increment(
; CHECK-IC1-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-IC1-NEXT:  [[ENTRY:.*:]]
; CHECK-IC1-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK-IC1:       [[VECTOR_PH]]:
; CHECK-IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-IC1:       [[VECTOR_BODY]]:
; CHECK-IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT:    [[TMP0:%.*]] = add i64 15, [[INDEX]]
; CHECK-IC1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-IC1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
; CHECK-IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP0]]
; CHECK-IC1-NEXT:    store <4 x i32> [[WIDE_LOAD]], ptr [[TMP2]], align 4
; CHECK-IC1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-IC1-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
; CHECK-IC1-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-IC1:       [[MIDDLE_BLOCK]]:
; CHECK-IC1-NEXT:    br label %[[EXIT:.*]]
; CHECK-IC1:       [[EXIT]]:
; CHECK-IC1-NEXT:    ret void
;
; CHECK-TF-LABEL: define void @test_unconditional_increment(
; CHECK-TF-SAME: ptr noalias writeonly [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-TF-NEXT:  [[ENTRY:.*:]]
; CHECK-TF-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK-TF:       [[VECTOR_PH]]:
; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-TF:       [[VECTOR_BODY]]:
; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[TMP0:%.*]] = add i64 15, [[INDEX]]
; CHECK-TF-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-TF-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
; CHECK-TF-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP0]]
; CHECK-TF-NEXT:    store <4 x i32> [[WIDE_LOAD]], ptr [[TMP2]], align 4
; CHECK-TF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-TF-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
; CHECK-TF-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-TF:       [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT:    br label %[[EXIT:.*]]
; CHECK-TF:       [[EXIT]]:
; CHECK-TF-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 15, %entry ], [ %idx.1, %for.inc ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br label %inc.step

inc.step:
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %idx
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %inc.step ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

define void @test_multiple_conditional_inductions(ptr %dst, ptr noalias %dst2, ptr noalias %src, ptr noalias %cond, ptr noalias %cond2) {
; CHECK-IC1-LABEL: define void @test_multiple_conditional_inductions(
; CHECK-IC1-SAME: ptr [[DST:%.*]], ptr noalias [[DST2:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[COND:%.*]], ptr noalias [[COND2:%.*]]) {
; CHECK-IC1-NEXT:  [[VECTOR_PH:.*]]:
; CHECK-IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-IC1:       [[VECTOR_BODY]]:
; CHECK-IC1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[EXIT:.*]] ]
; CHECK-IC1-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[DST_INC:%.*]], %[[EXIT]] ]
; CHECK-IC1-NEXT:    [[DST2_IDX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[DST2_INC:%.*]], %[[EXIT]] ]
; CHECK-IC1-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]]
; CHECK-IC1-NEXT:    [[COND_VAL:%.*]] = load i32, ptr [[TMP0]], align 4
; CHECK-IC1-NEXT:    [[COND_IS_ZERO:%.*]] = icmp eq i32 [[COND_VAL]], 0
; CHECK-IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-IC1-NEXT:    [[SRC_VAL:%.*]] = load i32, ptr [[TMP2]], align 4
; CHECK-IC1-NEXT:    br i1 [[COND_IS_ZERO]], label %[[IF_END:.*]], label %[[IF_THEN0:.*]]
; CHECK-IC1:       [[IF_THEN0]]:
; CHECK-IC1-NEXT:    [[DST_IDX_NEXT:%.*]] = add nsw i64 [[CONDITIONAL_IV]], 1
; CHECK-IC1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[CONDITIONAL_IV]]
; CHECK-IC1-NEXT:    [[DST_VAL_TRUNC:%.*]] = trunc i32 [[SRC_VAL]] to i8
; CHECK-IC1-NEXT:    store i8 [[DST_VAL_TRUNC]], ptr [[TMP3]], align 1
; CHECK-IC1-NEXT:    br label %[[IF_END]]
; CHECK-IC1:       [[IF_END]]:
; CHECK-IC1-NEXT:    [[DST_INC]] = phi i64 [ [[DST_IDX_NEXT]], %[[IF_THEN0]] ], [ [[CONDITIONAL_IV]], %[[VECTOR_BODY]] ]
; CHECK-IC1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[COND2]], i64 [[INDEX]]
; CHECK-IC1-NEXT:    [[COND2_VAL:%.*]] = load i32, ptr [[TMP7]], align 4
; CHECK-IC1-NEXT:    [[COND2_IS_ZERO:%.*]] = icmp eq i32 [[COND2_VAL]], 0
; CHECK-IC1-NEXT:    br i1 [[COND2_IS_ZERO]], label %[[EXIT]], label %[[MIDDLE_BLOCK:.*]]
; CHECK-IC1:       [[MIDDLE_BLOCK]]:
; CHECK-IC1-NEXT:    [[DST2_VAL_TRUNC:%.*]] = trunc i32 [[SRC_VAL]] to i16
; CHECK-IC1-NEXT:    [[DST2_IDX_NEXT:%.*]] = add nsw i64 [[DST2_IDX]], 1
; CHECK-IC1-NEXT:    [[DST2_GEP:%.*]] = getelementptr inbounds i16, ptr [[DST2]], i64 [[DST2_IDX]]
; CHECK-IC1-NEXT:    store i16 [[DST2_VAL_TRUNC]], ptr [[DST2_GEP]], align 2
; CHECK-IC1-NEXT:    br label %[[EXIT]]
; CHECK-IC1:       [[EXIT]]:
; CHECK-IC1-NEXT:    [[DST2_INC]] = phi i64 [ [[DST2_IDX_NEXT]], %[[MIDDLE_BLOCK]] ], [ [[DST2_IDX]], %[[IF_END]] ]
; CHECK-IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[INDEX]], 1
; CHECK-IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
; CHECK-IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]]
; CHECK-IC1:       [[EXIT1]]:
; CHECK-IC1-NEXT:    ret void
;
; CHECK-TF-LABEL: define void @test_multiple_conditional_inductions(
; CHECK-TF-SAME: ptr [[DST:%.*]], ptr noalias [[DST2:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[COND:%.*]], ptr noalias [[COND2:%.*]]) {
; CHECK-TF-NEXT:  [[VECTOR_PH:.*]]:
; CHECK-TF-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-TF:       [[VECTOR_BODY]]:
; CHECK-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[EXIT:.*]] ]
; CHECK-TF-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[DST_INC:%.*]], %[[EXIT]] ]
; CHECK-TF-NEXT:    [[DST2_IDX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[DST2_INC:%.*]], %[[EXIT]] ]
; CHECK-TF-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]]
; CHECK-TF-NEXT:    [[COND_VAL:%.*]] = load i32, ptr [[TMP0]], align 4
; CHECK-TF-NEXT:    [[COND_IS_ZERO:%.*]] = icmp eq i32 [[COND_VAL]], 0
; CHECK-TF-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-TF-NEXT:    [[SRC_VAL:%.*]] = load i32, ptr [[TMP2]], align 4
; CHECK-TF-NEXT:    br i1 [[COND_IS_ZERO]], label %[[IF_END:.*]], label %[[IF_THEN0:.*]]
; CHECK-TF:       [[IF_THEN0]]:
; CHECK-TF-NEXT:    [[DST_IDX_NEXT:%.*]] = add nsw i64 [[CONDITIONAL_IV]], 1
; CHECK-TF-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[CONDITIONAL_IV]]
; CHECK-TF-NEXT:    [[DST_VAL_TRUNC:%.*]] = trunc i32 [[SRC_VAL]] to i8
; CHECK-TF-NEXT:    store i8 [[DST_VAL_TRUNC]], ptr [[TMP3]], align 1
; CHECK-TF-NEXT:    br label %[[IF_END]]
; CHECK-TF:       [[IF_END]]:
; CHECK-TF-NEXT:    [[DST_INC]] = phi i64 [ [[DST_IDX_NEXT]], %[[IF_THEN0]] ], [ [[CONDITIONAL_IV]], %[[VECTOR_BODY]] ]
; CHECK-TF-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[COND2]], i64 [[INDEX]]
; CHECK-TF-NEXT:    [[COND2_VAL:%.*]] = load i32, ptr [[TMP7]], align 4
; CHECK-TF-NEXT:    [[COND2_IS_ZERO:%.*]] = icmp eq i32 [[COND2_VAL]], 0
; CHECK-TF-NEXT:    br i1 [[COND2_IS_ZERO]], label %[[EXIT]], label %[[MIDDLE_BLOCK:.*]]
; CHECK-TF:       [[MIDDLE_BLOCK]]:
; CHECK-TF-NEXT:    [[DST2_VAL_TRUNC:%.*]] = trunc i32 [[SRC_VAL]] to i16
; CHECK-TF-NEXT:    [[DST2_IDX_NEXT:%.*]] = add nsw i64 [[DST2_IDX]], 1
; CHECK-TF-NEXT:    [[DST2_GEP:%.*]] = getelementptr inbounds i16, ptr [[DST2]], i64 [[DST2_IDX]]
; CHECK-TF-NEXT:    store i16 [[DST2_VAL_TRUNC]], ptr [[DST2_GEP]], align 2
; CHECK-TF-NEXT:    br label %[[EXIT]]
; CHECK-TF:       [[EXIT]]:
; CHECK-TF-NEXT:    [[DST2_INC]] = phi i64 [ [[DST2_IDX_NEXT]], %[[MIDDLE_BLOCK]] ], [ [[DST2_IDX]], %[[IF_END]] ]
; CHECK-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[INDEX]], 1
; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]]
; CHECK-TF:       [[EXIT1]]:
; CHECK-TF-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %dst.idx = phi i64 [ 0, %entry ], [ %dst.inc, %for.inc ]
  %dst2.idx = phi i64 [ 0, %entry ], [ %dst2.inc, %for.inc ]
  %cond.gep = getelementptr inbounds i32, ptr %cond, i64 %iv
  %cond.val = load i32, ptr %cond.gep, align 4
  %cond.is.zero = icmp eq i32 %cond.val, 0
  %src.gep = getelementptr inbounds i32, ptr %src, i64 %iv
  %src.val = load i32, ptr %src.gep, align 4
  br i1 %cond.is.zero, label %if.end, label %if.then0

if.then0:
  %dst.idx.next = add nsw i64 %dst.idx, 1
  %dst.gep = getelementptr inbounds i8, ptr %dst, i64 %dst.idx
  %dst.val.trunc = trunc i32 %src.val to i8
  store i8 %dst.val.trunc, ptr %dst.gep, align 1
  br label %if.end

if.end:
  %dst.inc = phi i64 [ %dst.idx.next, %if.then0 ], [ %dst.idx, %for.body ]
  %cond2.gep = getelementptr inbounds i32, ptr %cond2, i64 %iv
  %cond2.val = load i32, ptr %cond2.gep, align 4
  %cond2.is.zero = icmp eq i32 %cond2.val, 0
  br i1 %cond2.is.zero, label %for.inc, label %if.then1

if.then1:
  %dst2.val.trunc = trunc i32 %src.val to i16
  %dst2.idx.next = add nsw i64 %dst2.idx, 1
  %dst2.gep = getelementptr inbounds i16, ptr %dst2, i64 %dst2.idx
  store i16 %dst2.val.trunc, ptr %dst2.gep, align 2
  br label %for.inc

for.inc:
  %dst2.inc = phi i64 [ %dst2.idx.next, %if.then1 ], [ %dst2.idx, %if.end ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

; Test expand load with always false update (this probably can be simplified).
define void @test_expand_load_always_false_cond(ptr noalias %dst, ptr readonly %src, i32 %c) {
; CHECK-IC1-LABEL: define void @test_expand_load_always_false_cond(
; CHECK-IC1-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-IC1-NEXT:  [[ENTRY:.*]]:
; CHECK-IC1-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK-IC1:       [[VECTOR_PH]]:
; CHECK-IC1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[EXIT:.*]] ]
; CHECK-IC1-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1:%.*]], %[[EXIT]] ]
; CHECK-IC1-NEXT:    [[DST_PTR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
; CHECK-IC1-NEXT:    [[LOAD_DST:%.*]] = load i32, ptr [[DST_PTR]], align 4
; CHECK-IC1-NEXT:    br i1 false, label %[[VECTOR_BODY:.*]], label %[[EXIT]]
; CHECK-IC1:       [[VECTOR_BODY]]:
; CHECK-IC1-NEXT:    [[TMP1:%.*]] = sext i32 [[CONDITIONAL_IV]] to i64
; CHECK-IC1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]
; CHECK-IC1-NEXT:    [[LOAD_SRC:%.*]] = load i32, ptr [[TMP2]], align 4
; CHECK-IC1-NEXT:    store i32 [[LOAD_SRC]], ptr [[DST_PTR]], align 4
; CHECK-IC1-NEXT:    [[IDX_NEXT:%.*]] = add nsw i32 [[CONDITIONAL_IV]], 1
; CHECK-IC1-NEXT:    br label %[[EXIT]]
; CHECK-IC1:       [[EXIT]]:
; CHECK-IC1-NEXT:    [[IDX_1]] = phi i32 [ [[IDX_NEXT]], %[[VECTOR_BODY]] ], [ [[CONDITIONAL_IV]], %[[VECTOR_PH]] ]
; CHECK-IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-IC1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
; CHECK-IC1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_PH]]
; CHECK-IC1:       [[EXIT1]]:
; CHECK-IC1-NEXT:    ret void
;
; CHECK-TF-LABEL: define void @test_expand_load_always_false_cond(
; CHECK-TF-SAME: ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[C:%.*]]) {
; CHECK-TF-NEXT:  [[ENTRY:.*]]:
; CHECK-TF-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK-TF:       [[VECTOR_PH]]:
; CHECK-TF-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[EXIT:.*]] ]
; CHECK-TF-NEXT:    [[CONDITIONAL_IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IDX_1:%.*]], %[[EXIT]] ]
; CHECK-TF-NEXT:    [[DST_PTR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
; CHECK-TF-NEXT:    [[LOAD_DST:%.*]] = load i32, ptr [[DST_PTR]], align 4
; CHECK-TF-NEXT:    br i1 false, label %[[VECTOR_BODY:.*]], label %[[EXIT]]
; CHECK-TF:       [[VECTOR_BODY]]:
; CHECK-TF-NEXT:    [[TMP1:%.*]] = sext i32 [[CONDITIONAL_IV]] to i64
; CHECK-TF-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]
; CHECK-TF-NEXT:    [[LOAD_SRC:%.*]] = load i32, ptr [[TMP2]], align 4
; CHECK-TF-NEXT:    store i32 [[LOAD_SRC]], ptr [[DST_PTR]], align 4
; CHECK-TF-NEXT:    [[IDX_NEXT:%.*]] = add nsw i32 [[CONDITIONAL_IV]], 1
; CHECK-TF-NEXT:    br label %[[EXIT]]
; CHECK-TF:       [[EXIT]]:
; CHECK-TF-NEXT:    [[IDX_1]] = phi i32 [ [[IDX_NEXT]], %[[VECTOR_BODY]] ], [ [[CONDITIONAL_IV]], %[[VECTOR_PH]] ]
; CHECK-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
; CHECK-TF-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT1:.*]], label %[[VECTOR_PH]]
; CHECK-TF:       [[EXIT1]]:
; CHECK-TF-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i32 [ 0, %entry ], [ %idx.1, %for.inc ]
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv
  %load.dst = load i32, ptr %dst.ptr, align 4
  br i1 0, label %if.then, label %for.inc

if.then:
  %src.idx = sext i32 %idx to i64
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %src.idx
  %load.src = load i32, ptr %src.ptr, align 4
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i32 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i32 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 4096
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}