llvm/test/Transforms/LoopVectorize/iv_outside_user.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --replace-value-regex "!llvm.loop ![0-9]+" --version 5
; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 < %s | FileCheck --check-prefixes=COMMON,NOTAILFOLD,VEC %s
; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 < %s | FileCheck --check-prefixes=COMMON,NOTAILFOLD,INTERLEAVE %s
; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -tail-folding-policy=must-fold-tail -force-tail-folding-style=data < %s | FileCheck --check-prefixes=COMMON,TAILFOLD %s

define i32 @postinc(i32 %k)  {
; NOTAILFOLD-LABEL: define i32 @postinc(
; NOTAILFOLD-SAME: i32 [[K:%.*]]) {
; NOTAILFOLD-NEXT:  [[ENTRY:.*]]:
; NOTAILFOLD-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
; NOTAILFOLD-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NOTAILFOLD:       [[VECTOR_PH]]:
; NOTAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[K]], 1
; NOTAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
; NOTAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; NOTAILFOLD:       [[VECTOR_BODY]]:
; NOTAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOTAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; NOTAILFOLD-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[MIDDLE_BLOCK]]:
; NOTAILFOLD-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; NOTAILFOLD:       [[SCALAR_PH]]:
; NOTAILFOLD-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; NOTAILFOLD-NEXT:    br label %[[FOR_BODY:.*]]
; NOTAILFOLD:       [[FOR_BODY]]:
; NOTAILFOLD-NEXT:    [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
; NOTAILFOLD-NEXT:    [[INC]] = add nsw i32 [[INC_PHI]], 1
; NOTAILFOLD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
; NOTAILFOLD-NEXT:    br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[FOR_END]]:
; NOTAILFOLD-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
; NOTAILFOLD-NEXT:    ret i32 [[INC_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @postinc(
; TAILFOLD-SAME: i32 [[K:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    [[N_RND_UP:%.*]] = add i32 [[K]], 1
; TAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    br label %[[FOR_END:.*]]
; TAILFOLD:       [[FOR_END]]:
; TAILFOLD-NEXT:    ret i32 [[K]]
;
entry:
  br label %for.body

for.body:
  %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ]
  %inc = add nsw i32 %inc.phi, 1
  %cmp = icmp eq i32 %inc, %k
  br i1 %cmp, label %for.end, label %for.body

for.end:
  ret i32 %inc
}

define i32 @preinc(i32 %k)  {
; NOTAILFOLD-LABEL: define i32 @preinc(
; NOTAILFOLD-SAME: i32 [[K:%.*]]) {
; NOTAILFOLD-NEXT:  [[ENTRY:.*]]:
; NOTAILFOLD-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
; NOTAILFOLD-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NOTAILFOLD:       [[VECTOR_PH]]:
; NOTAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[K]], 1
; NOTAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
; NOTAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; NOTAILFOLD:       [[VECTOR_BODY]]:
; NOTAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOTAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; NOTAILFOLD-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[MIDDLE_BLOCK]]:
; NOTAILFOLD-NEXT:    [[IND_ESCAPE:%.*]] = sub i32 [[N_VEC]], 1
; NOTAILFOLD-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; NOTAILFOLD:       [[SCALAR_PH]]:
; NOTAILFOLD-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; NOTAILFOLD-NEXT:    br label %[[FOR_BODY:.*]]
; NOTAILFOLD:       [[FOR_BODY]]:
; NOTAILFOLD-NEXT:    [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
; NOTAILFOLD-NEXT:    [[INC]] = add nsw i32 [[INC_PHI]], 1
; NOTAILFOLD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
; NOTAILFOLD-NEXT:    br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[FOR_END]]:
; NOTAILFOLD-NEXT:    [[INC_PHI_LCSSA:%.*]] = phi i32 [ [[INC_PHI]], %[[FOR_BODY]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK]] ]
; NOTAILFOLD-NEXT:    ret i32 [[INC_PHI_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @preinc(
; TAILFOLD-SAME: i32 [[K:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    [[N_RND_UP:%.*]] = add i32 [[K]], 1
; TAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    [[IND_ESCAPE:%.*]] = sub i32 [[K]], 1
; TAILFOLD-NEXT:    br label %[[FOR_END:.*]]
; TAILFOLD:       [[FOR_END]]:
; TAILFOLD-NEXT:    ret i32 [[IND_ESCAPE]]
;
entry:
  br label %for.body

for.body:
  %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ]
  %inc = add nsw i32 %inc.phi, 1
  %cmp = icmp eq i32 %inc, %k
  br i1 %cmp, label %for.end, label %for.body

for.end:
  ret i32 %inc.phi
}

define i32 @constpre()  {
; COMMON-LABEL: define i32 @constpre() {
; COMMON-NEXT:  [[ENTRY:.*:]]
; COMMON-NEXT:    br label %[[VECTOR_PH:.*]]
; COMMON:       [[VECTOR_PH]]:
; COMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMMON:       [[VECTOR_BODY]]:
; COMMON-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMMON-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; COMMON-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
; COMMON-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; COMMON:       [[MIDDLE_BLOCK]]:
; COMMON-NEXT:    br label %[[FOR_END:.*]]
; COMMON:       [[FOR_END]]:
; COMMON-NEXT:    ret i32 2
;
entry:
  br label %for.body

for.body:
  %inc.phi = phi i32 [ 32, %entry ], [ %inc, %for.body ]
  %inc = sub nsw i32 %inc.phi, 2
  %cmp = icmp eq i32 %inc, 0
  br i1 %cmp, label %for.end, label %for.body

for.end:
  ret i32 %inc.phi
}

define ptr @geppre(ptr %ptr) {
; COMMON-LABEL: define ptr @geppre(
; COMMON-SAME: ptr [[PTR:%.*]]) {
; COMMON-NEXT:  [[ENTRY:.*:]]
; COMMON-NEXT:    br label %[[VECTOR_PH:.*]]
; COMMON:       [[VECTOR_PH]]:
; COMMON-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 512
; COMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMMON:       [[VECTOR_BODY]]:
; COMMON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMMON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; COMMON-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
; COMMON-NEXT:    br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; COMMON:       [[MIDDLE_BLOCK]]:
; COMMON-NEXT:    [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP0]], i64 -16
; COMMON-NEXT:    br label %[[FOR_END:.*]]
; COMMON:       [[FOR_END]]:
; COMMON-NEXT:    ret ptr [[IND_ESCAPE]]
;
entry:
  br label %for.body

for.body:
  %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ]
  %ptr.phi = phi ptr [ %ptr, %entry ], [ %inc.ptr, %for.body ]
  %inc = add nsw i32 %inc.phi, 1
  %inc.ptr = getelementptr i32, ptr %ptr.phi, i32 4
  %cmp = icmp eq i32 %inc, 32
  br i1 %cmp, label %for.end, label %for.body

for.end:
  ret ptr %ptr.phi
}

define ptr @both(ptr %p, i32 %k)  {
; VEC-LABEL: define ptr @both(
; VEC-SAME: ptr [[P:%.*]], i32 [[K:%.*]]) {
; VEC-NEXT:  [[ENTRY:.*]]:
; VEC-NEXT:    [[BASE:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 1
; VEC-NEXT:    [[TMP0:%.*]] = add i32 [[K]], -1
; VEC-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
; VEC-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; VEC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2
; VEC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VEC:       [[VECTOR_PH]]:
; VEC-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[TMP2]], 1
; VEC-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
; VEC-NEXT:    [[TMP3:%.*]] = trunc i64 [[N_VEC]] to i32
; VEC-NEXT:    [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
; VEC-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP4]]
; VEC-NEXT:    br label %[[VECTOR_BODY:.*]]
; VEC:       [[VECTOR_BODY]]:
; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[BASE]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <2 x i64> <i64 0, i64 4>
; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 8
; VEC-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VEC-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; VEC:       [[MIDDLE_BLOCK]]:
; VEC-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x ptr> [[VECTOR_GEP]], i64 1
; VEC-NEXT:    [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP5]], i64 -4
; VEC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
; VEC-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; VEC:       [[SCALAR_PH]]:
; VEC-NEXT:    [[TMP9:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; VEC-NEXT:    [[TMP10:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
; VEC-NEXT:    [[TMP11:%.*]] = phi ptr [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
; VEC-NEXT:    br label %[[FOR_BODY:.*]]
; VEC:       [[FOR_BODY]]:
; VEC-NEXT:    [[INC_PHI:%.*]] = phi i32 [ [[TMP9]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
; VEC-NEXT:    [[INC_LAG1:%.*]] = phi ptr [ [[TMP10]], %[[SCALAR_PH]] ], [ [[TMP:%.*]], %[[FOR_BODY]] ]
; VEC-NEXT:    [[INC_LAG2:%.*]] = phi ptr [ [[TMP11]], %[[SCALAR_PH]] ], [ [[INC_LAG1]], %[[FOR_BODY]] ]
; VEC-NEXT:    [[TMP]] = getelementptr inbounds i32, ptr [[INC_LAG1]], i64 1
; VEC-NEXT:    [[INC]] = add nsw i32 [[INC_PHI]], 1
; VEC-NEXT:    [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
; VEC-NEXT:    br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
; VEC:       [[FOR_END]]:
; VEC-NEXT:    [[INC_LAG1_LCSSA:%.*]] = phi ptr [ [[INC_LAG1]], %[[FOR_BODY]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK]] ]
; VEC-NEXT:    ret ptr [[INC_LAG1_LCSSA]]
;
; INTERLEAVE-LABEL: define ptr @both(
; INTERLEAVE-SAME: ptr [[P:%.*]], i32 [[K:%.*]]) {
; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
; INTERLEAVE-NEXT:    [[BASE:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 1
; INTERLEAVE-NEXT:    [[TMP0:%.*]] = add i32 [[K]], -1
; INTERLEAVE-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
; INTERLEAVE-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; INTERLEAVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2
; INTERLEAVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTERLEAVE:       [[VECTOR_PH]]:
; INTERLEAVE-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[TMP2]], 1
; INTERLEAVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
; INTERLEAVE-NEXT:    [[TMP3:%.*]] = trunc i64 [[N_VEC]] to i32
; INTERLEAVE-NEXT:    [[TMP8:%.*]] = shl i64 [[N_VEC]], 2
; INTERLEAVE-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP8]]
; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
; INTERLEAVE:       [[VECTOR_BODY]]:
; INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; INTERLEAVE-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTERLEAVE-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[MIDDLE_BLOCK]]:
; INTERLEAVE-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
; INTERLEAVE-NEXT:    [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 4
; INTERLEAVE-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP9]]
; INTERLEAVE-NEXT:    [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i64 -4
; INTERLEAVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
; INTERLEAVE-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; INTERLEAVE:       [[SCALAR_PH]]:
; INTERLEAVE-NEXT:    [[TMP10:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTERLEAVE-NEXT:    [[TMP11:%.*]] = phi ptr [ [[NEXT_GEP1]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
; INTERLEAVE-NEXT:    [[TMP12:%.*]] = phi ptr [ [[NEXT_GEP]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
; INTERLEAVE-NEXT:    br label %[[FOR_BODY:.*]]
; INTERLEAVE:       [[FOR_BODY]]:
; INTERLEAVE-NEXT:    [[INC_PHI:%.*]] = phi i32 [ [[TMP10]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
; INTERLEAVE-NEXT:    [[INC_LAG1:%.*]] = phi ptr [ [[TMP11]], %[[SCALAR_PH]] ], [ [[TMP:%.*]], %[[FOR_BODY]] ]
; INTERLEAVE-NEXT:    [[INC_LAG2:%.*]] = phi ptr [ [[TMP12]], %[[SCALAR_PH]] ], [ [[INC_LAG1]], %[[FOR_BODY]] ]
; INTERLEAVE-NEXT:    [[TMP]] = getelementptr inbounds i32, ptr [[INC_LAG1]], i64 1
; INTERLEAVE-NEXT:    [[INC]] = add nsw i32 [[INC_PHI]], 1
; INTERLEAVE-NEXT:    [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
; INTERLEAVE-NEXT:    br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[FOR_END]]:
; INTERLEAVE-NEXT:    [[INC_LAG1_LCSSA:%.*]] = phi ptr [ [[INC_LAG1]], %[[FOR_BODY]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT:    ret ptr [[INC_LAG1_LCSSA]]
;
; TAILFOLD-LABEL: define ptr @both(
; TAILFOLD-SAME: ptr [[P:%.*]], i32 [[K:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    [[BASE:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 1
; TAILFOLD-NEXT:    [[TMP0:%.*]] = add i32 [[K]], -1
; TAILFOLD-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
; TAILFOLD-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP2]], 1
; TAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 1
; TAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 2
; TAILFOLD-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP3]]
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP4]], i64 -4
; TAILFOLD-NEXT:    br label %[[FOR_END:.*]]
; TAILFOLD:       [[FOR_END]]:
; TAILFOLD-NEXT:    ret ptr [[IND_ESCAPE]]
;
entry:
  %base = getelementptr inbounds i32, ptr %p, i64 1
  br label %for.body

for.body:
  %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ]
  %inc.lag1 = phi ptr [ %base, %entry ], [ %tmp, %for.body]
  %inc.lag2 = phi ptr [ %base, %entry ], [ %inc.lag1, %for.body]
  %tmp = getelementptr inbounds i32, ptr %inc.lag1, i64 1
  %inc = add nsw i32 %inc.phi, 1
  %cmp = icmp eq i32 %inc, %k
  br i1 %cmp, label %for.end, label %for.body

for.end:
  ret ptr %inc.lag1
}

define i32 @multiphi(i32 %k, ptr %p)  {
; NOTAILFOLD-LABEL: define i32 @multiphi(
; NOTAILFOLD-SAME: i32 [[K:%.*]], ptr [[P:%.*]]) {
; NOTAILFOLD-NEXT:  [[ENTRY:.*]]:
; NOTAILFOLD-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
; NOTAILFOLD-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NOTAILFOLD:       [[VECTOR_PH]]:
; NOTAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[K]], 1
; NOTAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
; NOTAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; NOTAILFOLD:       [[VECTOR_BODY]]:
; NOTAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOTAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; NOTAILFOLD-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[MIDDLE_BLOCK]]:
; NOTAILFOLD-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; NOTAILFOLD:       [[SCALAR_PH]]:
; NOTAILFOLD-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; NOTAILFOLD-NEXT:    br label %[[FOR_BODY:.*]]
; NOTAILFOLD:       [[FOR_BODY]]:
; NOTAILFOLD-NEXT:    [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
; NOTAILFOLD-NEXT:    [[INC]] = add nsw i32 [[INC_PHI]], 1
; NOTAILFOLD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
; NOTAILFOLD-NEXT:    br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[FOR_END]]:
; NOTAILFOLD-NEXT:    [[PHI:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
; NOTAILFOLD-NEXT:    [[PHI2:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
; NOTAILFOLD-NEXT:    store i32 [[PHI2]], ptr [[P]], align 4
; NOTAILFOLD-NEXT:    ret i32 [[PHI]]
;
; TAILFOLD-LABEL: define i32 @multiphi(
; TAILFOLD-SAME: i32 [[K:%.*]], ptr [[P:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    [[N_RND_UP:%.*]] = add i32 [[K]], 1
; TAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    br label %[[FOR_END:.*]]
; TAILFOLD:       [[FOR_END]]:
; TAILFOLD-NEXT:    store i32 [[K]], ptr [[P]], align 4
; TAILFOLD-NEXT:    ret i32 [[K]]
;
entry:
  br label %for.body

for.body:
  %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ]
  %inc = add nsw i32 %inc.phi, 1
  %cmp = icmp eq i32 %inc, %k
  br i1 %cmp, label %for.end, label %for.body

for.end:
  %phi = phi i32 [ %inc, %for.body ]
  %phi2 = phi i32 [ %inc, %for.body ]
  store i32 %phi2, ptr %p
  ret i32 %phi
}

define void @PR30742(ptr %p) {
; NOTAILFOLD-LABEL: define void @PR30742(
; NOTAILFOLD-SAME: ptr [[P:%.*]]) {
; NOTAILFOLD-NEXT:  [[BB0:.*:]]
; NOTAILFOLD-NEXT:    br label %[[BB1:.*]]
; NOTAILFOLD:       [[BB1_LOOPEXIT:.*]]:
; NOTAILFOLD-NEXT:    br label %[[BB1]]
; NOTAILFOLD:       [[BB1]]:
; NOTAILFOLD-NEXT:    [[TMP00:%.*]] = load i32, ptr [[P]], align 16
; NOTAILFOLD-NEXT:    [[TMP01:%.*]] = sub i32 [[TMP00]], 3
; NOTAILFOLD-NEXT:    [[TMP02:%.*]] = icmp slt i32 [[TMP01]], 1
; NOTAILFOLD-NEXT:    [[TMP03:%.*]] = select i1 [[TMP02]], i32 1, i32 [[TMP01]]
; NOTAILFOLD-NEXT:    [[TMP04:%.*]] = add nsw i32 [[TMP03]], -7
; NOTAILFOLD-NEXT:    [[TMP1:%.*]] = add nsw i32 [[TMP03]], -15
; NOTAILFOLD-NEXT:    [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP1]], i32 0)
; NOTAILFOLD-NEXT:    [[TMP7:%.*]] = add i32 [[TMP03]], -8
; NOTAILFOLD-NEXT:    [[TMP2:%.*]] = sub i32 [[TMP7]], [[SMIN1]]
; NOTAILFOLD-NEXT:    [[TMP3:%.*]] = lshr i32 [[TMP2]], 3
; NOTAILFOLD-NEXT:    [[TMP4:%.*]] = add nuw nsw i32 [[TMP3]], 1
; NOTAILFOLD-NEXT:    [[MIN_ITERS_CHECK4:%.*]] = icmp ult i32 [[TMP4]], 2
; NOTAILFOLD-NEXT:    br i1 [[MIN_ITERS_CHECK4]], label %[[SCALAR_PH2:.*]], label %[[VECTOR_PH4:.*]]
; NOTAILFOLD:       [[VECTOR_PH4]]:
; NOTAILFOLD-NEXT:    [[N_MOD_VF6:%.*]] = and i32 [[TMP4]], 1
; NOTAILFOLD-NEXT:    [[N_VEC7:%.*]] = sub i32 [[TMP4]], [[N_MOD_VF6]]
; NOTAILFOLD-NEXT:    [[TMP5:%.*]] = mul i32 [[N_VEC7]], -8
; NOTAILFOLD-NEXT:    [[IND_END8:%.*]] = add i32 [[TMP04]], [[TMP5]]
; NOTAILFOLD-NEXT:    br label %[[VECTOR_BODY7:.*]]
; NOTAILFOLD:       [[VECTOR_BODY7]]:
; NOTAILFOLD-NEXT:    [[INDEX10:%.*]] = phi i32 [ 0, %[[VECTOR_PH4]] ], [ [[INDEX_NEXT11:%.*]], %[[VECTOR_BODY7]] ]
; NOTAILFOLD-NEXT:    [[INDEX_NEXT11]] = add nuw i32 [[INDEX10]], 2
; NOTAILFOLD-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT11]], [[N_VEC7]]
; NOTAILFOLD-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK10:.*]], label %[[VECTOR_BODY7]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[MIDDLE_BLOCK10]]:
; NOTAILFOLD-NEXT:    [[IND_ESCAPE:%.*]] = sub i32 [[IND_END8]], -8
; NOTAILFOLD-NEXT:    [[CMP_N12:%.*]] = icmp eq i32 [[TMP4]], [[N_VEC7]]
; NOTAILFOLD-NEXT:    br i1 [[CMP_N12]], label %[[BB3:.*]], label %[[SCALAR_PH2]]
; NOTAILFOLD:       [[SCALAR_PH2]]:
; NOTAILFOLD-NEXT:    [[BC_RESUME_VAL10:%.*]] = phi i32 [ [[IND_END8]], %[[MIDDLE_BLOCK10]] ], [ [[TMP04]], %[[BB1]] ]
; NOTAILFOLD-NEXT:    br label %[[BB2:.*]]
; NOTAILFOLD:       [[BB2]]:
; NOTAILFOLD-NEXT:    [[TMP05:%.*]] = phi i32 [ [[BC_RESUME_VAL10]], %[[SCALAR_PH2]] ], [ [[TMP06:%.*]], %[[BB2]] ]
; NOTAILFOLD-NEXT:    [[TMP06]] = add i32 [[TMP05]], -8
; NOTAILFOLD-NEXT:    [[LOOP1_EC:%.*]] = icmp sgt i32 [[TMP06]], 0
; NOTAILFOLD-NEXT:    br i1 [[LOOP1_EC]], label %[[BB2]], label %[[BB3]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[BB3]]:
; NOTAILFOLD-NEXT:    [[IV1_LCSSA:%.*]] = phi i32 [ [[TMP05]], %[[BB2]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK10]] ]
; NOTAILFOLD-NEXT:    [[TMP09:%.*]] = sub i32 [[TMP00]], 4
; NOTAILFOLD-NEXT:    [[TMP10:%.*]] = icmp slt i32 [[TMP09]], 1
; NOTAILFOLD-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32 1, i32 [[TMP09]]
; NOTAILFOLD-NEXT:    [[TMP12:%.*]] = add nsw i32 [[TMP11]], -7
; NOTAILFOLD-NEXT:    [[TMP8:%.*]] = add nsw i32 [[TMP11]], -15
; NOTAILFOLD-NEXT:    [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP8]], i32 0)
; NOTAILFOLD-NEXT:    [[TMP18:%.*]] = add i32 [[TMP11]], -8
; NOTAILFOLD-NEXT:    [[TMP21:%.*]] = sub i32 [[TMP18]], [[SMIN]]
; NOTAILFOLD-NEXT:    [[TMP20:%.*]] = lshr i32 [[TMP21]], 3
; NOTAILFOLD-NEXT:    [[TMP14:%.*]] = add nuw nsw i32 [[TMP20]], 1
; NOTAILFOLD-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP14]], 2
; NOTAILFOLD-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NOTAILFOLD:       [[VECTOR_PH]]:
; NOTAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[TMP14]], 1
; NOTAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP14]], [[N_MOD_VF]]
; NOTAILFOLD-NEXT:    [[TMP16:%.*]] = mul i32 [[N_VEC]], -8
; NOTAILFOLD-NEXT:    [[IND_END:%.*]] = add i32 [[TMP12]], [[TMP16]]
; NOTAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; NOTAILFOLD:       [[VECTOR_BODY]]:
; NOTAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOTAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; NOTAILFOLD-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[MIDDLE_BLOCK]]:
; NOTAILFOLD-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP14]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[CMP_N]], label %[[BB1_LOOPEXIT]], label %[[SCALAR_PH]]
; NOTAILFOLD:       [[SCALAR_PH]]:
; NOTAILFOLD-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[TMP12]], %[[BB3]] ]
; NOTAILFOLD-NEXT:    br label %[[BB4:.*]]
; NOTAILFOLD:       [[BB4]]:
; NOTAILFOLD-NEXT:    [[IV2:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], %[[BB4]] ]
; NOTAILFOLD-NEXT:    [[IV2_NEXT]] = add i32 [[IV2]], -8
; NOTAILFOLD-NEXT:    [[LOOP2_EC:%.*]] = icmp sgt i32 [[IV2_NEXT]], 0
; NOTAILFOLD-NEXT:    br i1 [[LOOP2_EC]], label %[[BB4]], label %[[BB1_LOOPEXIT]], {{!llvm.loop ![0-9]+}}
;
; TAILFOLD-LABEL: define void @PR30742(
; TAILFOLD-SAME: ptr [[P:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    br label %[[OUTER:.*]]
; TAILFOLD:       [[OUTER_LOOPEXIT:.*]]:
; TAILFOLD-NEXT:    br label %[[OUTER]]
; TAILFOLD:       [[OUTER]]:
; TAILFOLD-NEXT:    [[N:%.*]] = load i32, ptr [[P]], align 16
; TAILFOLD-NEXT:    [[TMP0:%.*]] = sub i32 [[N]], 3
; TAILFOLD-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 1
; TAILFOLD-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 1, i32 [[TMP0]]
; TAILFOLD-NEXT:    [[START1:%.*]] = add nsw i32 [[TMP2]], -7
; TAILFOLD-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -15
; TAILFOLD-NEXT:    [[TMP4:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP3]], i32 0)
; TAILFOLD-NEXT:    [[TMP5:%.*]] = add i32 [[TMP2]], -8
; TAILFOLD-NEXT:    [[TMP6:%.*]] = sub i32 [[TMP5]], [[TMP4]]
; TAILFOLD-NEXT:    [[TMP7:%.*]] = lshr i32 [[TMP6]], 3
; TAILFOLD-NEXT:    [[TMP8:%.*]] = add nuw nsw i32 [[TMP7]], 1
; TAILFOLD-NEXT:    br label %[[VECTOR_PH1:.*]]
; TAILFOLD:       [[VECTOR_PH1]]:
; TAILFOLD-NEXT:    [[N_RND_UP2:%.*]] = add i32 [[TMP8]], 1
; TAILFOLD-NEXT:    [[N_MOD_VF3:%.*]] = and i32 [[N_RND_UP2]], 1
; TAILFOLD-NEXT:    [[N_VEC4:%.*]] = sub i32 [[N_RND_UP2]], [[N_MOD_VF3]]
; TAILFOLD-NEXT:    [[TMP9:%.*]] = mul i32 [[TMP8]], -8
; TAILFOLD-NEXT:    [[TMP10:%.*]] = add i32 [[START1]], [[TMP9]]
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY5:.*]]
; TAILFOLD:       [[VECTOR_BODY5]]:
; TAILFOLD-NEXT:    [[INDEX6:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT7:%.*]], %[[VECTOR_BODY5]] ]
; TAILFOLD-NEXT:    [[INDEX_NEXT7]] = add nuw i32 [[INDEX6]], 2
; TAILFOLD-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT7]], [[N_VEC4]]
; TAILFOLD-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK8:.*]], label %[[VECTOR_BODY5]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK8]]:
; TAILFOLD-NEXT:    [[IND_ESCAPE:%.*]] = sub i32 [[TMP10]], -8
; TAILFOLD-NEXT:    br label %[[LOOP1_EXIT:.*]]
; TAILFOLD:       [[LOOP1_EXIT]]:
; TAILFOLD-NEXT:    [[TMP12:%.*]] = sub i32 [[N]], 4
; TAILFOLD-NEXT:    [[TMP13:%.*]] = icmp slt i32 [[TMP12]], 1
; TAILFOLD-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 1, i32 [[TMP12]]
; TAILFOLD-NEXT:    [[START2:%.*]] = add nsw i32 [[TMP14]], -7
; TAILFOLD-NEXT:    [[TMP15:%.*]] = add nsw i32 [[TMP14]], -15
; TAILFOLD-NEXT:    [[TMP16:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP15]], i32 0)
; TAILFOLD-NEXT:    [[TMP22:%.*]] = add i32 [[TMP14]], -8
; TAILFOLD-NEXT:    [[TMP18:%.*]] = sub i32 [[TMP22]], [[TMP16]]
; TAILFOLD-NEXT:    [[TMP19:%.*]] = lshr i32 [[TMP18]], 3
; TAILFOLD-NEXT:    [[TMP20:%.*]] = add nuw nsw i32 [[TMP19]], 1
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP20]], 1
; TAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    br label %[[OUTER_LOOPEXIT]]
;
entry:
  br label %outer

outer:
  %n = load i32, ptr %p, align 16
  %0 = sub i32 %n, 3
  %1 = icmp slt i32 %0, 1
  %2 = select i1 %1, i32 1, i32 %0
  %start1 = add nsw i32 %2, -7
  br label %loop1

loop1:
  %iv1 = phi i32 [ %start1, %outer ], [ %iv1.next, %loop1 ]
  %iv1.next = add i32 %iv1, -8
  %loop1.ec = icmp sgt i32 %iv1.next, 0
  br i1 %loop1.ec, label %loop1, label %loop1.exit

loop1.exit:
  %iv1.lcssa = phi i32 [ %iv1, %loop1 ]
  %3 = sub i32 %n, 4
  %4 = icmp slt i32 %3, 1
  %5 = select i1 %4, i32 1, i32 %3
  %start2 = add nsw i32 %5, -7
  br label %loop2

loop2:
  %iv2 = phi i32 [ %start2, %loop1.exit ], [ %iv2.next, %loop2 ]
  %iv2.next = add i32 %iv2, -8
  %loop2.ec = icmp sgt i32 %iv2.next, 0
  br i1 %loop2.ec, label %loop2, label %outer
}

;
define i64 @iv_scalar_steps_and_outside_users(ptr %ptr) {
; VEC-LABEL: define i64 @iv_scalar_steps_and_outside_users(
; VEC-SAME: ptr [[PTR:%.*]]) {
; VEC-NEXT:  [[ENTRY:.*:]]
; VEC-NEXT:    br label %[[VECTOR_PH:.*]]
; VEC:       [[VECTOR_PH]]:
; VEC-NEXT:    br label %[[VECTOR_BODY:.*]]
; VEC:       [[VECTOR_BODY]]:
; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[INDEX]]
; VEC-NEXT:    store <2 x i64> [[VEC_IND]], ptr [[TMP1]], align 4
; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT:    [[VEC_IND_NEXT]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
; VEC-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1002
; VEC-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; VEC:       [[MIDDLE_BLOCK]]:
; VEC-NEXT:    br label %[[EXIT:.*]]
; VEC:       [[EXIT]]:
; VEC-NEXT:    ret i64 1001
;
; INTERLEAVE-LABEL: define i64 @iv_scalar_steps_and_outside_users(
; INTERLEAVE-SAME: ptr [[PTR:%.*]]) {
; INTERLEAVE-NEXT:  [[ENTRY:.*:]]
; INTERLEAVE-NEXT:    br label %[[VECTOR_PH:.*]]
; INTERLEAVE:       [[VECTOR_PH]]:
; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
; INTERLEAVE:       [[VECTOR_BODY]]:
; INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTERLEAVE-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
; INTERLEAVE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[INDEX]]
; INTERLEAVE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[TMP1]]
; INTERLEAVE-NEXT:    store i64 [[INDEX]], ptr [[TMP2]], align 4
; INTERLEAVE-NEXT:    store i64 [[TMP1]], ptr [[TMP3]], align 4
; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; INTERLEAVE-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1002
; INTERLEAVE-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[MIDDLE_BLOCK]]:
; INTERLEAVE-NEXT:    br label %[[EXIT:.*]]
; INTERLEAVE:       [[EXIT]]:
; INTERLEAVE-NEXT:    ret i64 1001
;
; TAILFOLD-LABEL: define i64 @iv_scalar_steps_and_outside_users(
; TAILFOLD-SAME: ptr [[PTR:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TAILFOLD-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TAILFOLD-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[INDEX]]
; TAILFOLD-NEXT:    store <2 x i64> [[VEC_IND]], ptr [[TMP0]], align 4
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; TAILFOLD-NEXT:    [[VEC_IND_NEXT]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
; TAILFOLD-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1002
; TAILFOLD-NEXT:    br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    br label %[[EXIT:.*]]
; TAILFOLD:       [[EXIT]]:
; TAILFOLD-NEXT:    ret i64 1001
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %iv.next = add nuw i64 %iv, 1
  %gep.ptr = getelementptr inbounds i64, ptr %ptr, i64 %iv
  store i64 %iv, ptr %gep.ptr
  %exitcond = icmp ugt i64 %iv, 1000
  br i1 %exitcond, label %exit, label %loop

exit:
  %iv.lcssa = phi i64 [ %iv, %loop ]
  ret i64 %iv.lcssa
}


; %iv.2 is dead in the vector loop and only used outside the loop.
define i32 @iv_2_dead_in_loop_only_used_outside(ptr %ptr) {
; VEC-LABEL: define i32 @iv_2_dead_in_loop_only_used_outside(
; VEC-SAME: ptr [[PTR:%.*]]) {
; VEC-NEXT:  [[ENTRY:.*:]]
; VEC-NEXT:    br label %[[VECTOR_PH:.*]]
; VEC:       [[VECTOR_PH]]:
; VEC-NEXT:    br label %[[VECTOR_BODY:.*]]
; VEC:       [[VECTOR_BODY]]:
; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[INDEX]]
; VEC-NEXT:    store <2 x i64> [[VEC_IND]], ptr [[TMP1]], align 4
; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT:    [[VEC_IND_NEXT]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
; VEC-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1002
; VEC-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; VEC:       [[MIDDLE_BLOCK]]:
; VEC-NEXT:    br label %[[EXIT:.*]]
; VEC:       [[EXIT]]:
; VEC-NEXT:    ret i32 2002
;
; INTERLEAVE-LABEL: define i32 @iv_2_dead_in_loop_only_used_outside(
; INTERLEAVE-SAME: ptr [[PTR:%.*]]) {
; INTERLEAVE-NEXT:  [[ENTRY:.*:]]
; INTERLEAVE-NEXT:    br label %[[VECTOR_PH:.*]]
; INTERLEAVE:       [[VECTOR_PH]]:
; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
; INTERLEAVE:       [[VECTOR_BODY]]:
; INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTERLEAVE-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
; INTERLEAVE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[INDEX]]
; INTERLEAVE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[TMP1]]
; INTERLEAVE-NEXT:    store i64 [[INDEX]], ptr [[TMP2]], align 4
; INTERLEAVE-NEXT:    store i64 [[TMP1]], ptr [[TMP3]], align 4
; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; INTERLEAVE-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1002
; INTERLEAVE-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[MIDDLE_BLOCK]]:
; INTERLEAVE-NEXT:    br label %[[EXIT:.*]]
; INTERLEAVE:       [[EXIT]]:
; INTERLEAVE-NEXT:    ret i32 2002
;
; TAILFOLD-LABEL: define i32 @iv_2_dead_in_loop_only_used_outside(
; TAILFOLD-SAME: ptr [[PTR:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TAILFOLD-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TAILFOLD-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[INDEX]]
; TAILFOLD-NEXT:    store <2 x i64> [[VEC_IND]], ptr [[TMP0]], align 4
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; TAILFOLD-NEXT:    [[VEC_IND_NEXT]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
; TAILFOLD-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1002
; TAILFOLD-NEXT:    br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    br label %[[EXIT:.*]]
; TAILFOLD:       [[EXIT]]:
; TAILFOLD-NEXT:    ret i32 2002
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %iv.2 = phi i32 [ 0, %entry ], [ %iv.2.next, %loop ]
  %iv.next = add nuw i64 %iv, 1
  %iv.2.next = add nuw i32 %iv.2, 2
  %gep.ptr = getelementptr inbounds i64, ptr %ptr, i64 %iv
  store i64 %iv, ptr %gep.ptr
  %exitcond = icmp ugt i64 %iv, 1000
  br i1 %exitcond, label %exit, label %loop

exit:
  %iv.2.lcssa = phi i32 [ %iv.2, %loop ]
  ret i32 %iv.2.lcssa
}

define i32 @postinc_sub(i32 %k)  {
; NOTAILFOLD-LABEL: define i32 @postinc_sub(
; NOTAILFOLD-SAME: i32 [[K:%.*]]) {
; NOTAILFOLD-NEXT:  [[ENTRY:.*]]:
; NOTAILFOLD-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
; NOTAILFOLD-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NOTAILFOLD:       [[VECTOR_PH]]:
; NOTAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[K]], 1
; NOTAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
; NOTAILFOLD-NEXT:    [[TMP0:%.*]] = sub i32 [[K]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; NOTAILFOLD:       [[VECTOR_BODY]]:
; NOTAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOTAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; NOTAILFOLD-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[MIDDLE_BLOCK]]:
; NOTAILFOLD-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; NOTAILFOLD:       [[SCALAR_PH]]:
; NOTAILFOLD-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ [[K]], %[[ENTRY]] ]
; NOTAILFOLD-NEXT:    br label %[[FOR_BODY:.*]]
; NOTAILFOLD:       [[FOR_BODY]]:
; NOTAILFOLD-NEXT:    [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
; NOTAILFOLD-NEXT:    [[INC]] = sub nsw i32 [[INC_PHI]], 1
; NOTAILFOLD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[INC]], 0
; NOTAILFOLD-NEXT:    br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[FOR_END]]:
; NOTAILFOLD-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[TMP0]], %[[MIDDLE_BLOCK]] ]
; NOTAILFOLD-NEXT:    ret i32 [[INC_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @postinc_sub(
; TAILFOLD-SAME: i32 [[K:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    [[N_RND_UP:%.*]] = add i32 [[K]], 1
; TAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT:    br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    br label %[[FOR_END:.*]]
; TAILFOLD:       [[FOR_END]]:
; TAILFOLD-NEXT:    ret i32 0
;
entry:
  br label %for.body

for.body:
  %inc.phi = phi i32 [ %k, %entry ], [ %inc, %for.body ]
  %inc = sub nsw i32 %inc.phi, 1
  %cmp = icmp eq i32 %inc, 0
  br i1 %cmp, label %for.end, label %for.body

for.end:
  ret i32 %inc
}

define i32 @postinc_swapped_ops(i32 %k)  {
; NOTAILFOLD-LABEL: define i32 @postinc_swapped_ops(
; NOTAILFOLD-SAME: i32 [[K:%.*]]) {
; NOTAILFOLD-NEXT:  [[ENTRY:.*]]:
; NOTAILFOLD-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
; NOTAILFOLD-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NOTAILFOLD:       [[VECTOR_PH]]:
; NOTAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[K]], 1
; NOTAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
; NOTAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; NOTAILFOLD:       [[VECTOR_BODY]]:
; NOTAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOTAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; NOTAILFOLD-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[MIDDLE_BLOCK]]:
; NOTAILFOLD-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; NOTAILFOLD:       [[SCALAR_PH]]:
; NOTAILFOLD-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; NOTAILFOLD-NEXT:    br label %[[FOR_BODY:.*]]
; NOTAILFOLD:       [[FOR_BODY]]:
; NOTAILFOLD-NEXT:    [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
; NOTAILFOLD-NEXT:    [[INC]] = add nsw i32 1, [[INC_PHI]]
; NOTAILFOLD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
; NOTAILFOLD-NEXT:    br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[FOR_END]]:
; NOTAILFOLD-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ]
; NOTAILFOLD-NEXT:    ret i32 [[INC_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @postinc_swapped_ops(
; TAILFOLD-SAME: i32 [[K:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    [[N_RND_UP:%.*]] = add i32 [[K]], 1
; TAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    br label %[[FOR_END:.*]]
; TAILFOLD:       [[FOR_END]]:
; TAILFOLD-NEXT:    ret i32 [[K]]
;
entry:
  br label %for.body

for.body:
  %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ]
  %inc = add nsw i32 1, %inc.phi
  %cmp = icmp eq i32 %inc, %k
  br i1 %cmp, label %for.end, label %for.body

for.end:
  ret i32 %inc
}

define i32 @postinc_not_iv_backedge_value(i32 %k)  {
; NOTAILFOLD-LABEL: define i32 @postinc_not_iv_backedge_value(
; NOTAILFOLD-SAME: i32 [[K:%.*]]) {
; NOTAILFOLD-NEXT:  [[ENTRY:.*]]:
; NOTAILFOLD-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2
; NOTAILFOLD-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NOTAILFOLD:       [[VECTOR_PH]]:
; NOTAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[K]], 1
; NOTAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]]
; NOTAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; NOTAILFOLD:       [[VECTOR_BODY]]:
; NOTAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOTAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; NOTAILFOLD-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[MIDDLE_BLOCK]]:
; NOTAILFOLD-NEXT:    [[TMP1:%.*]] = sub nuw i32 [[N_VEC]], 1
; NOTAILFOLD-NEXT:    [[TMP2:%.*]] = add i32 2, [[TMP1]]
; NOTAILFOLD-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; NOTAILFOLD:       [[SCALAR_PH]]:
; NOTAILFOLD-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; NOTAILFOLD-NEXT:    br label %[[FOR_BODY:.*]]
; NOTAILFOLD:       [[FOR_BODY]]:
; NOTAILFOLD-NEXT:    [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
; NOTAILFOLD-NEXT:    [[INC]] = add nsw i32 [[INC_PHI]], 1
; NOTAILFOLD-NEXT:    [[INC_2:%.*]] = add i32 [[INC_PHI]], 2
; NOTAILFOLD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]]
; NOTAILFOLD-NEXT:    br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[FOR_END]]:
; NOTAILFOLD-NEXT:    [[INC_2_LCSSA:%.*]] = phi i32 [ [[INC_2]], %[[FOR_BODY]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ]
; NOTAILFOLD-NEXT:    ret i32 [[INC_2_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @postinc_not_iv_backedge_value(
; TAILFOLD-SAME: i32 [[K:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    [[N_RND_UP:%.*]] = add i32 [[K]], 1
; TAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    [[TMP1:%.*]] = sub nuw i32 [[K]], 1
; TAILFOLD-NEXT:    [[TMP3:%.*]] = add i32 2, [[TMP1]]
; TAILFOLD-NEXT:    br label %[[FOR_END:.*]]
; TAILFOLD:       [[FOR_END]]:
; TAILFOLD-NEXT:    ret i32 [[TMP3]]
;
entry:
  br label %for.body

for.body:
  %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ]
  %inc = add nsw i32 %inc.phi, 1
  %inc.2 = add i32 %inc.phi, 2
  %cmp = icmp eq i32 %inc, %k
  br i1 %cmp, label %for.end, label %for.body

for.end:
  ret i32 %inc.2
}

define ptr @postinc_not_iv_backedge_value_ptr_induction(ptr %p.init)  {
; COMMON-LABEL: define ptr @postinc_not_iv_backedge_value_ptr_induction(
; COMMON-SAME: ptr [[P_INIT:%.*]]) {
; COMMON-NEXT:  [[ENTRY:.*:]]
; COMMON-NEXT:    [[P_END:%.*]] = getelementptr nusw nuw i8, ptr [[P_INIT]], i64 1024
; COMMON-NEXT:    br label %[[VECTOR_PH:.*]]
; COMMON:       [[VECTOR_PH]]:
; COMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMMON:       [[VECTOR_BODY]]:
; COMMON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMMON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; COMMON-NEXT:    [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; COMMON-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; COMMON:       [[MIDDLE_BLOCK]]:
; COMMON-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 2
; COMMON-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i64 1023
; COMMON-NEXT:    br label %[[END:.*]]
; COMMON:       [[END]]:
; COMMON-NEXT:    ret ptr [[TMP2]]
;
entry:
  %p.end = getelementptr nuw nusw i8, ptr %p.init, i64 1024
  br label %loop

loop:
  %p.iv = phi ptr [ %p.init, %entry ], [ %p.next, %loop ]
  %p.next = getelementptr nuw i8, ptr %p.iv, i64 1
  %ret = getelementptr i8, ptr %p.iv, i64 2
  %ec = icmp eq ptr %p.next, %p.end
  br i1 %ec, label %end, label %loop

end:
  ret ptr %ret
}

define ptr @postinc_not_iv_backedge_value_ptr_runtime_tc(ptr %p.init, i64 %n)  {
; NOTAILFOLD-LABEL: define ptr @postinc_not_iv_backedge_value_ptr_runtime_tc(
; NOTAILFOLD-SAME: ptr [[P_INIT:%.*]], i64 [[N:%.*]]) {
; NOTAILFOLD-NEXT:  [[ENTRY:.*]]:
; NOTAILFOLD-NEXT:    [[P_END:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 [[N]]
; NOTAILFOLD-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
; NOTAILFOLD-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; NOTAILFOLD:       [[VECTOR_PH]]:
; NOTAILFOLD-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 1
; NOTAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; NOTAILFOLD-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 [[N_VEC]]
; NOTAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; NOTAILFOLD:       [[VECTOR_BODY]]:
; NOTAILFOLD-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NOTAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; NOTAILFOLD-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[MIDDLE_BLOCK]]:
; NOTAILFOLD-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 2
; NOTAILFOLD-NEXT:    [[TMP4:%.*]] = sub nuw i64 [[N_VEC]], 1
; NOTAILFOLD-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP4]]
; NOTAILFOLD-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; NOTAILFOLD-NEXT:    br i1 [[CMP_N]], label %[[END:.*]], label %[[SCALAR_PH]]
; NOTAILFOLD:       [[SCALAR_PH]]:
; NOTAILFOLD-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[P_INIT]], %[[ENTRY]] ]
; NOTAILFOLD-NEXT:    br label %[[LOOP:.*]]
; NOTAILFOLD:       [[LOOP]]:
; NOTAILFOLD-NEXT:    [[P_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
; NOTAILFOLD-NEXT:    [[P_NEXT]] = getelementptr nuw i8, ptr [[P_IV]], i64 1
; NOTAILFOLD-NEXT:    [[RET:%.*]] = getelementptr i8, ptr [[P_IV]], i64 2
; NOTAILFOLD-NEXT:    [[EC:%.*]] = icmp eq ptr [[P_NEXT]], [[P_END]]
; NOTAILFOLD-NEXT:    br i1 [[EC]], label %[[END]], label %[[LOOP]], {{!llvm.loop ![0-9]+}}
; NOTAILFOLD:       [[END]]:
; NOTAILFOLD-NEXT:    [[RET_LCSSA:%.*]] = phi ptr [ [[RET]], %[[LOOP]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ]
; NOTAILFOLD-NEXT:    ret ptr [[RET_LCSSA]]
;
; TAILFOLD-LABEL: define ptr @postinc_not_iv_backedge_value_ptr_runtime_tc(
; TAILFOLD-SAME: ptr [[P_INIT:%.*]], i64 [[N:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    [[P_END:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 [[N]]
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], 1
; TAILFOLD-NEXT:    [[TMP0:%.*]] = and i64 [[N_RND_UP]], 1
; TAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP0]]
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT:    br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[P_INIT]], i64 2
; TAILFOLD-NEXT:    [[TMP3:%.*]] = sub nuw i64 [[N]], 1
; TAILFOLD-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP2]], i64 [[TMP3]]
; TAILFOLD-NEXT:    br label %[[END:.*]]
; TAILFOLD:       [[END]]:
; TAILFOLD-NEXT:    ret ptr [[TMP4]]
;
entry:
  %p.end = getelementptr i8, ptr %p.init, i64 %n
  br label %loop

loop:
  %p.iv = phi ptr [ %p.init, %entry ], [ %p.next, %loop ]
  %p.next = getelementptr nuw i8, ptr %p.iv, i64 1
  %ret = getelementptr i8, ptr %p.iv, i64 2
  %ec = icmp eq ptr %p.next, %p.end
  br i1 %ec, label %end, label %loop

end:
  ret ptr %ret
}

define float @fp_postinc_use_fadd(float %init, ptr noalias nocapture %A, i64 %N, float %fpinc) {
; VEC-LABEL: define float @fp_postinc_use_fadd(
; VEC-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
; VEC-NEXT:  [[ENTRY:.*]]:
; VEC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
; VEC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VEC:       [[VECTOR_PH]]:
; VEC-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N]], 1
; VEC-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; VEC-NEXT:    [[DOTCAST:%.*]] = sitofp i64 [[N_VEC]] to float
; VEC-NEXT:    [[TMP0:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
; VEC-NEXT:    [[TMP1:%.*]] = fadd fast float [[INIT]], [[TMP0]]
; VEC-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0
; VEC-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
; VEC-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <2 x float> poison, float [[FPINC]], i64 0
; VEC-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT1]], <2 x float> poison, <2 x i32> zeroinitializer
; VEC-NEXT:    [[TMP2:%.*]] = fmul fast <2 x float> <float 0.000000e+00, float 1.000000e+00>, [[DOTSPLAT2]]
; VEC-NEXT:    [[INDUCTION:%.*]] = fadd fast <2 x float> [[DOTSPLAT]], [[TMP2]]
; VEC-NEXT:    [[TMP3:%.*]] = fmul fast float [[FPINC]], 2.000000e+00
; VEC-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <2 x float> poison, float [[TMP3]], i64 0
; VEC-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT3]], <2 x float> poison, <2 x i32> zeroinitializer
; VEC-NEXT:    br label %[[VECTOR_BODY:.*]]
; VEC:       [[VECTOR_BODY]]:
; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
; VEC-NEXT:    store <2 x float> [[VEC_IND]], ptr [[TMP5]], align 4
; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT:    [[VEC_IND_NEXT]] = fadd fast <2 x float> [[VEC_IND]], [[DOTSPLAT4]]
; VEC-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VEC-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; VEC:       [[MIDDLE_BLOCK]]:
; VEC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; VEC-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VEC:       [[SCALAR_PH]]:
; VEC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; VEC-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi float [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
; VEC-NEXT:    br label %[[LOOP:.*]]
; VEC:       [[LOOP]]:
; VEC-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; VEC-NEXT:    [[FP_IV:%.*]] = phi float [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ]
; VEC-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
; VEC-NEXT:    store float [[FP_IV]], ptr [[GEP_A]], align 4
; VEC-NEXT:    [[ADD]] = fadd fast float [[FP_IV]], [[FPINC]]
; VEC-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; VEC-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; VEC-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}}
; VEC:       [[EXIT]]:
; VEC-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
; VEC-NEXT:    ret float [[ADD_LCSSA]]
;
; INTERLEAVE-LABEL: define float @fp_postinc_use_fadd(
; INTERLEAVE-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
; INTERLEAVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
; INTERLEAVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTERLEAVE:       [[VECTOR_PH]]:
; INTERLEAVE-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N]], 1
; INTERLEAVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; INTERLEAVE-NEXT:    [[DOTCAST:%.*]] = sitofp i64 [[N_VEC]] to float
; INTERLEAVE-NEXT:    [[TMP0:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
; INTERLEAVE-NEXT:    [[TMP1:%.*]] = fadd fast float [[INIT]], [[TMP0]]
; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
; INTERLEAVE:       [[VECTOR_BODY]]:
; INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTERLEAVE-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 1
; INTERLEAVE-NEXT:    [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float
; INTERLEAVE-NEXT:    [[TMP4:%.*]] = fmul fast float [[FPINC]], [[DOTCAST1]]
; INTERLEAVE-NEXT:    [[OFFSET_IDX:%.*]] = fadd fast float [[INIT]], [[TMP4]]
; INTERLEAVE-NEXT:    [[TMP7:%.*]] = fmul fast float 1.000000e+00, [[FPINC]]
; INTERLEAVE-NEXT:    [[TMP8:%.*]] = fadd fast float [[OFFSET_IDX]], [[TMP7]]
; INTERLEAVE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
; INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP3]]
; INTERLEAVE-NEXT:    store float [[OFFSET_IDX]], ptr [[TMP6]], align 4
; INTERLEAVE-NEXT:    store float [[TMP8]], ptr [[TMP10]], align 4
; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; INTERLEAVE-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTERLEAVE-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[MIDDLE_BLOCK]]:
; INTERLEAVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; INTERLEAVE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; INTERLEAVE:       [[SCALAR_PH]]:
; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTERLEAVE-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi float [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
; INTERLEAVE:       [[LOOP]]:
; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; INTERLEAVE-NEXT:    [[FP_IV:%.*]] = phi float [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ]
; INTERLEAVE-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
; INTERLEAVE-NEXT:    store float [[FP_IV]], ptr [[GEP_A]], align 4
; INTERLEAVE-NEXT:    [[ADD]] = fadd fast float [[FP_IV]], [[FPINC]]
; INTERLEAVE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; INTERLEAVE-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; INTERLEAVE-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[EXIT]]:
; INTERLEAVE-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT:    ret float [[ADD_LCSSA]]
;
; TAILFOLD-LABEL: define float @fp_postinc_use_fadd(
; TAILFOLD-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], 1
; TAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 1
; TAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT:    [[TMP0:%.*]] = sitofp i64 [[N]] to float
; TAILFOLD-NEXT:    [[TMP1:%.*]] = fmul fast float [[FPINC]], [[TMP0]]
; TAILFOLD-NEXT:    [[TMP2:%.*]] = fadd fast float [[INIT]], [[TMP1]]
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
; TAILFOLD-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[N]])
; TAILFOLD-NEXT:    [[TMP3:%.*]] = sitofp i64 [[INDEX]] to float
; TAILFOLD-NEXT:    [[TMP4:%.*]] = fmul fast float [[FPINC]], [[TMP3]]
; TAILFOLD-NEXT:    [[TMP5:%.*]] = fadd fast float [[INIT]], [[TMP4]]
; TAILFOLD-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
; TAILFOLD-NEXT:    br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; TAILFOLD:       [[PRED_STORE_IF]]:
; TAILFOLD-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
; TAILFOLD-NEXT:    store float [[TMP5]], ptr [[TMP7]], align 4
; TAILFOLD-NEXT:    br label %[[PRED_STORE_CONTINUE]]
; TAILFOLD:       [[PRED_STORE_CONTINUE]]:
; TAILFOLD-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
; TAILFOLD-NEXT:    br i1 [[TMP8]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; TAILFOLD:       [[PRED_STORE_IF1]]:
; TAILFOLD-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 1
; TAILFOLD-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
; TAILFOLD-NEXT:    [[TMP11:%.*]] = fmul fast float 1.000000e+00, [[FPINC]]
; TAILFOLD-NEXT:    [[TMP12:%.*]] = fadd fast float [[TMP5]], [[TMP11]]
; TAILFOLD-NEXT:    store float [[TMP12]], ptr [[TMP10]], align 4
; TAILFOLD-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
; TAILFOLD:       [[PRED_STORE_CONTINUE2]]:
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    br label %[[EXIT:.*]]
; TAILFOLD:       [[EXIT]]:
; TAILFOLD-NEXT:    ret float [[TMP2]]
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ %init, %entry ], [ %add, %loop ]
  %gep.A = getelementptr inbounds float, ptr %A, i64 %iv
  store float %fp.iv, ptr %gep.A, align 4
  %add = fadd fast float %fp.iv, %fpinc
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %N
  br i1 %ec, label %exit, label %loop

exit:
  ret float %add
}

define float @fp_postinc_use_fadd_ops_swapped(float %init, ptr noalias nocapture %A, i64 %N, float %fpinc) {
; VEC-LABEL: define float @fp_postinc_use_fadd_ops_swapped(
; VEC-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
; VEC-NEXT:  [[ENTRY:.*]]:
; VEC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
; VEC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VEC:       [[VECTOR_PH]]:
; VEC-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N]], 1
; VEC-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; VEC-NEXT:    [[DOTCAST:%.*]] = sitofp i64 [[N_VEC]] to float
; VEC-NEXT:    [[TMP0:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
; VEC-NEXT:    [[TMP1:%.*]] = fadd fast float [[INIT]], [[TMP0]]
; VEC-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0
; VEC-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
; VEC-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <2 x float> poison, float [[FPINC]], i64 0
; VEC-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT1]], <2 x float> poison, <2 x i32> zeroinitializer
; VEC-NEXT:    [[TMP2:%.*]] = fmul fast <2 x float> <float 0.000000e+00, float 1.000000e+00>, [[DOTSPLAT2]]
; VEC-NEXT:    [[INDUCTION:%.*]] = fadd fast <2 x float> [[DOTSPLAT]], [[TMP2]]
; VEC-NEXT:    [[TMP3:%.*]] = fmul fast float [[FPINC]], 2.000000e+00
; VEC-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <2 x float> poison, float [[TMP3]], i64 0
; VEC-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT3]], <2 x float> poison, <2 x i32> zeroinitializer
; VEC-NEXT:    br label %[[VECTOR_BODY:.*]]
; VEC:       [[VECTOR_BODY]]:
; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
; VEC-NEXT:    store <2 x float> [[VEC_IND]], ptr [[TMP5]], align 4
; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT:    [[VEC_IND_NEXT]] = fadd fast <2 x float> [[VEC_IND]], [[DOTSPLAT4]]
; VEC-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VEC-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; VEC:       [[MIDDLE_BLOCK]]:
; VEC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; VEC-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VEC:       [[SCALAR_PH]]:
; VEC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; VEC-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi float [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
; VEC-NEXT:    br label %[[LOOP:.*]]
; VEC:       [[LOOP]]:
; VEC-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; VEC-NEXT:    [[FP_IV:%.*]] = phi float [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ]
; VEC-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
; VEC-NEXT:    store float [[FP_IV]], ptr [[GEP_A]], align 4
; VEC-NEXT:    [[ADD]] = fadd fast float [[FPINC]], [[FP_IV]]
; VEC-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; VEC-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; VEC-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}}
; VEC:       [[EXIT]]:
; VEC-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
; VEC-NEXT:    ret float [[ADD_LCSSA]]
;
; INTERLEAVE-LABEL: define float @fp_postinc_use_fadd_ops_swapped(
; INTERLEAVE-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
; INTERLEAVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
; INTERLEAVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTERLEAVE:       [[VECTOR_PH]]:
; INTERLEAVE-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N]], 1
; INTERLEAVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; INTERLEAVE-NEXT:    [[DOTCAST:%.*]] = sitofp i64 [[N_VEC]] to float
; INTERLEAVE-NEXT:    [[TMP0:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
; INTERLEAVE-NEXT:    [[TMP1:%.*]] = fadd fast float [[INIT]], [[TMP0]]
; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
; INTERLEAVE:       [[VECTOR_BODY]]:
; INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTERLEAVE-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 1
; INTERLEAVE-NEXT:    [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float
; INTERLEAVE-NEXT:    [[TMP4:%.*]] = fmul fast float [[FPINC]], [[DOTCAST1]]
; INTERLEAVE-NEXT:    [[OFFSET_IDX:%.*]] = fadd fast float [[INIT]], [[TMP4]]
; INTERLEAVE-NEXT:    [[TMP7:%.*]] = fmul fast float 1.000000e+00, [[FPINC]]
; INTERLEAVE-NEXT:    [[TMP8:%.*]] = fadd fast float [[OFFSET_IDX]], [[TMP7]]
; INTERLEAVE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
; INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP3]]
; INTERLEAVE-NEXT:    store float [[OFFSET_IDX]], ptr [[TMP6]], align 4
; INTERLEAVE-NEXT:    store float [[TMP8]], ptr [[TMP10]], align 4
; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; INTERLEAVE-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTERLEAVE-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[MIDDLE_BLOCK]]:
; INTERLEAVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; INTERLEAVE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; INTERLEAVE:       [[SCALAR_PH]]:
; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTERLEAVE-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi float [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
; INTERLEAVE:       [[LOOP]]:
; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; INTERLEAVE-NEXT:    [[FP_IV:%.*]] = phi float [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ]
; INTERLEAVE-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
; INTERLEAVE-NEXT:    store float [[FP_IV]], ptr [[GEP_A]], align 4
; INTERLEAVE-NEXT:    [[ADD]] = fadd fast float [[FPINC]], [[FP_IV]]
; INTERLEAVE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; INTERLEAVE-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; INTERLEAVE-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[EXIT]]:
; INTERLEAVE-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT:    ret float [[ADD_LCSSA]]
;
; TAILFOLD-LABEL: define float @fp_postinc_use_fadd_ops_swapped(
; TAILFOLD-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], 1
; TAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 1
; TAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT:    [[TMP0:%.*]] = sitofp i64 [[N]] to float
; TAILFOLD-NEXT:    [[TMP1:%.*]] = fmul fast float [[FPINC]], [[TMP0]]
; TAILFOLD-NEXT:    [[TMP2:%.*]] = fadd fast float [[INIT]], [[TMP1]]
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
; TAILFOLD-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[N]])
; TAILFOLD-NEXT:    [[TMP3:%.*]] = sitofp i64 [[INDEX]] to float
; TAILFOLD-NEXT:    [[TMP4:%.*]] = fmul fast float [[FPINC]], [[TMP3]]
; TAILFOLD-NEXT:    [[TMP5:%.*]] = fadd fast float [[INIT]], [[TMP4]]
; TAILFOLD-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
; TAILFOLD-NEXT:    br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; TAILFOLD:       [[PRED_STORE_IF]]:
; TAILFOLD-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
; TAILFOLD-NEXT:    store float [[TMP5]], ptr [[TMP7]], align 4
; TAILFOLD-NEXT:    br label %[[PRED_STORE_CONTINUE]]
; TAILFOLD:       [[PRED_STORE_CONTINUE]]:
; TAILFOLD-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
; TAILFOLD-NEXT:    br i1 [[TMP8]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; TAILFOLD:       [[PRED_STORE_IF1]]:
; TAILFOLD-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 1
; TAILFOLD-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
; TAILFOLD-NEXT:    [[TMP11:%.*]] = fmul fast float 1.000000e+00, [[FPINC]]
; TAILFOLD-NEXT:    [[TMP12:%.*]] = fadd fast float [[TMP5]], [[TMP11]]
; TAILFOLD-NEXT:    store float [[TMP12]], ptr [[TMP10]], align 4
; TAILFOLD-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
; TAILFOLD:       [[PRED_STORE_CONTINUE2]]:
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    br label %[[EXIT:.*]]
; TAILFOLD:       [[EXIT]]:
; TAILFOLD-NEXT:    ret float [[TMP2]]
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ %init, %entry ], [ %add, %loop ]
  %gep.A = getelementptr inbounds float, ptr %A, i64 %iv
  store float %fp.iv, ptr %gep.A, align 4
  %add = fadd fast float %fpinc, %fp.iv
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %N
  br i1 %ec, label %exit, label %loop

exit:
  ret float %add
}

define float @fp_postinc_use_fsub(float %init, ptr noalias nocapture %A, i64 %N, float %fpinc) {
; VEC-LABEL: define float @fp_postinc_use_fsub(
; VEC-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
; VEC-NEXT:  [[ENTRY:.*]]:
; VEC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
; VEC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VEC:       [[VECTOR_PH]]:
; VEC-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N]], 1
; VEC-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; VEC-NEXT:    [[DOTCAST:%.*]] = sitofp i64 [[N_VEC]] to float
; VEC-NEXT:    [[TMP0:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
; VEC-NEXT:    [[TMP1:%.*]] = fsub fast float [[INIT]], [[TMP0]]
; VEC-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0
; VEC-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
; VEC-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <2 x float> poison, float [[FPINC]], i64 0
; VEC-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT1]], <2 x float> poison, <2 x i32> zeroinitializer
; VEC-NEXT:    [[TMP2:%.*]] = fmul fast <2 x float> <float 0.000000e+00, float 1.000000e+00>, [[DOTSPLAT2]]
; VEC-NEXT:    [[INDUCTION:%.*]] = fsub fast <2 x float> [[DOTSPLAT]], [[TMP2]]
; VEC-NEXT:    [[TMP3:%.*]] = fmul fast float [[FPINC]], 2.000000e+00
; VEC-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <2 x float> poison, float [[TMP3]], i64 0
; VEC-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT3]], <2 x float> poison, <2 x i32> zeroinitializer
; VEC-NEXT:    br label %[[VECTOR_BODY:.*]]
; VEC:       [[VECTOR_BODY]]:
; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
; VEC-NEXT:    store <2 x float> [[VEC_IND]], ptr [[TMP5]], align 4
; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT:    [[VEC_IND_NEXT]] = fsub fast <2 x float> [[VEC_IND]], [[DOTSPLAT4]]
; VEC-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VEC-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; VEC:       [[MIDDLE_BLOCK]]:
; VEC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; VEC-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VEC:       [[SCALAR_PH]]:
; VEC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; VEC-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi float [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
; VEC-NEXT:    br label %[[LOOP:.*]]
; VEC:       [[LOOP]]:
; VEC-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; VEC-NEXT:    [[FP_IV:%.*]] = phi float [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ]
; VEC-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
; VEC-NEXT:    store float [[FP_IV]], ptr [[GEP_A]], align 4
; VEC-NEXT:    [[ADD]] = fsub fast float [[FP_IV]], [[FPINC]]
; VEC-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; VEC-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; VEC-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}}
; VEC:       [[EXIT]]:
; VEC-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
; VEC-NEXT:    ret float [[ADD_LCSSA]]
;
; INTERLEAVE-LABEL: define float @fp_postinc_use_fsub(
; INTERLEAVE-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
; INTERLEAVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
; INTERLEAVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTERLEAVE:       [[VECTOR_PH]]:
; INTERLEAVE-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N]], 1
; INTERLEAVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; INTERLEAVE-NEXT:    [[DOTCAST:%.*]] = sitofp i64 [[N_VEC]] to float
; INTERLEAVE-NEXT:    [[TMP0:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]]
; INTERLEAVE-NEXT:    [[TMP1:%.*]] = fsub fast float [[INIT]], [[TMP0]]
; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
; INTERLEAVE:       [[VECTOR_BODY]]:
; INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTERLEAVE-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 1
; INTERLEAVE-NEXT:    [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float
; INTERLEAVE-NEXT:    [[TMP4:%.*]] = fmul fast float [[FPINC]], [[DOTCAST1]]
; INTERLEAVE-NEXT:    [[OFFSET_IDX:%.*]] = fsub fast float [[INIT]], [[TMP4]]
; INTERLEAVE-NEXT:    [[TMP7:%.*]] = fmul fast float 1.000000e+00, [[FPINC]]
; INTERLEAVE-NEXT:    [[TMP8:%.*]] = fsub fast float [[OFFSET_IDX]], [[TMP7]]
; INTERLEAVE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
; INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP3]]
; INTERLEAVE-NEXT:    store float [[OFFSET_IDX]], ptr [[TMP6]], align 4
; INTERLEAVE-NEXT:    store float [[TMP8]], ptr [[TMP10]], align 4
; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; INTERLEAVE-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTERLEAVE-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[MIDDLE_BLOCK]]:
; INTERLEAVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; INTERLEAVE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; INTERLEAVE:       [[SCALAR_PH]]:
; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTERLEAVE-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi float [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ]
; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
; INTERLEAVE:       [[LOOP]]:
; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; INTERLEAVE-NEXT:    [[FP_IV:%.*]] = phi float [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ]
; INTERLEAVE-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
; INTERLEAVE-NEXT:    store float [[FP_IV]], ptr [[GEP_A]], align 4
; INTERLEAVE-NEXT:    [[ADD]] = fsub fast float [[FP_IV]], [[FPINC]]
; INTERLEAVE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; INTERLEAVE-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; INTERLEAVE-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[EXIT]]:
; INTERLEAVE-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT:    ret float [[ADD_LCSSA]]
;
; TAILFOLD-LABEL: define float @fp_postinc_use_fsub(
; TAILFOLD-SAME: float [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], 1
; TAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 1
; TAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT:    [[TMP0:%.*]] = sitofp i64 [[N]] to float
; TAILFOLD-NEXT:    [[TMP1:%.*]] = fmul fast float [[FPINC]], [[TMP0]]
; TAILFOLD-NEXT:    [[TMP2:%.*]] = fsub fast float [[INIT]], [[TMP1]]
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
; TAILFOLD-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[N]])
; TAILFOLD-NEXT:    [[TMP3:%.*]] = sitofp i64 [[INDEX]] to float
; TAILFOLD-NEXT:    [[TMP4:%.*]] = fmul fast float [[FPINC]], [[TMP3]]
; TAILFOLD-NEXT:    [[TMP5:%.*]] = fsub fast float [[INIT]], [[TMP4]]
; TAILFOLD-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
; TAILFOLD-NEXT:    br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; TAILFOLD:       [[PRED_STORE_IF]]:
; TAILFOLD-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
; TAILFOLD-NEXT:    store float [[TMP5]], ptr [[TMP7]], align 4
; TAILFOLD-NEXT:    br label %[[PRED_STORE_CONTINUE]]
; TAILFOLD:       [[PRED_STORE_CONTINUE]]:
; TAILFOLD-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
; TAILFOLD-NEXT:    br i1 [[TMP8]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; TAILFOLD:       [[PRED_STORE_IF1]]:
; TAILFOLD-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 1
; TAILFOLD-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
; TAILFOLD-NEXT:    [[TMP11:%.*]] = fmul fast float 1.000000e+00, [[FPINC]]
; TAILFOLD-NEXT:    [[TMP12:%.*]] = fsub fast float [[TMP5]], [[TMP11]]
; TAILFOLD-NEXT:    store float [[TMP12]], ptr [[TMP10]], align 4
; TAILFOLD-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
; TAILFOLD:       [[PRED_STORE_CONTINUE2]]:
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    br label %[[EXIT:.*]]
; TAILFOLD:       [[EXIT]]:
; TAILFOLD-NEXT:    ret float [[TMP2]]
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ %init, %entry ], [ %add, %loop ]
  %gep.A = getelementptr inbounds float, ptr %A, i64 %iv
  store float %fp.iv, ptr %gep.A, align 4
  %add = fsub fast float %fp.iv, %fpinc
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %N
  br i1 %ec, label %exit, label %loop

exit:
  ret float %add
}

; Test case for https://github.com/llvm/llvm-project/issues/121745.
define i32 @test_iv_uniform_with_outside_use_scev_simplification(ptr %dst) {
; VEC-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification(
; VEC-SAME: ptr [[DST:%.*]]) {
; VEC-NEXT:  [[ENTRY:.*:]]
; VEC-NEXT:    [[STEP_1:%.*]] = sext i8 0 to i32
; VEC-NEXT:    [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
; VEC-NEXT:    br label %[[VECTOR_PH:.*]]
; VEC:       [[VECTOR_PH]]:
; VEC-NEXT:    br label %[[VECTOR_BODY:.*]]
; VEC:       [[VECTOR_BODY]]:
; VEC-NEXT:    [[TMP0:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP0]]
; VEC-NEXT:    store <2 x i16> zeroinitializer, ptr [[TMP1]], align 2
; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[TMP0]], 2
; VEC-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
; VEC-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; VEC:       [[MIDDLE_BLOCK]]:
; VEC-NEXT:    br label %[[E_EXIT:.*]]
; VEC:       [[E_EXIT]]:
; VEC-NEXT:    ret i32 8
;
; INTERLEAVE-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification(
; INTERLEAVE-SAME: ptr [[DST:%.*]]) {
; INTERLEAVE-NEXT:  [[ENTRY:.*:]]
; INTERLEAVE-NEXT:    [[STEP_1:%.*]] = sext i8 0 to i32
; INTERLEAVE-NEXT:    [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
; INTERLEAVE-NEXT:    br label %[[VECTOR_PH:.*]]
; INTERLEAVE:       [[VECTOR_PH]]:
; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
; INTERLEAVE:       [[VECTOR_BODY]]:
; INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTERLEAVE-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
; INTERLEAVE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[INDEX]]
; INTERLEAVE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP1]]
; INTERLEAVE-NEXT:    store i16 0, ptr [[TMP2]], align 2
; INTERLEAVE-NEXT:    store i16 0, ptr [[TMP3]], align 2
; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; INTERLEAVE-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
; INTERLEAVE-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[MIDDLE_BLOCK]]:
; INTERLEAVE-NEXT:    br label %[[E_EXIT:.*]]
; INTERLEAVE:       [[E_EXIT]]:
; INTERLEAVE-NEXT:    ret i32 8
;
; TAILFOLD-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification(
; TAILFOLD-SAME: ptr [[DST:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    [[STEP_1:%.*]] = sext i8 0 to i32
; TAILFOLD-NEXT:    [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; TAILFOLD-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[INDEX]]
; TAILFOLD-NEXT:    store <2 x i16> zeroinitializer, ptr [[TMP0]], align 2
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
; TAILFOLD-NEXT:    br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    br label %[[E_EXIT:.*]]
; TAILFOLD:       [[E_EXIT]]:
; TAILFOLD-NEXT:    ret i32 8
;
entry:
  %step.1 = sext i8 0 to i32
  %step.2 = add nsw i32 %step.1, 1
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
  %gep.dst = getelementptr inbounds i16, ptr %dst, i32 %iv
  store i16 0, ptr %gep.dst, align 2
  %iv.next = add i32 %step.2, %iv
  %cmp.i = icmp slt i32 %iv.next, 8
  br i1 %cmp.i, label %loop, label %e.exit

e.exit:
  %res = phi i32 [ %iv.next, %loop ]
  ret i32 %res
}

define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(ptr %dst) {
; COMMON-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(
; COMMON-SAME: ptr [[DST:%.*]]) {
; COMMON-NEXT:  [[ENTRY:.*:]]
; COMMON-NEXT:    [[STEP_1:%.*]] = sext i8 0 to i32
; COMMON-NEXT:    [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
; COMMON-NEXT:    br label %[[VECTOR_PH:.*]]
; COMMON:       [[VECTOR_PH]]:
; COMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMMON:       [[VECTOR_BODY]]:
; COMMON-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMMON-NEXT:    [[TMP0:%.*]] = shl i32 [[INDEX]], 1
; COMMON-NEXT:    [[TMP1:%.*]] = add i32 [[TMP0]], 2
; COMMON-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP0]]
; COMMON-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP1]]
; COMMON-NEXT:    store i16 0, ptr [[TMP2]], align 2
; COMMON-NEXT:    store i16 0, ptr [[TMP3]], align 2
; COMMON-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; COMMON-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4
; COMMON-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; COMMON:       [[MIDDLE_BLOCK]]:
; COMMON-NEXT:    br label %[[E_EXIT:.*]]
; COMMON:       [[E_EXIT]]:
; COMMON-NEXT:    ret i32 8
;
entry:
  %step.1 = sext i8 0 to i32
  %step.2 = add nsw i32 %step.1, 1
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
  %gep.dst = getelementptr inbounds i16, ptr %dst, i32 %iv
  store i16 0, ptr %gep.dst, align 2
  %inc = add i32 %iv, 1
  %iv.next = add i32 %step.2, %inc
  %cmp.i = icmp slt i32 %iv.next, 8
  br i1 %cmp.i, label %loop, label %e.exit

e.exit:
  %res = phi i32 [ %iv.next, %loop ]
  ret i32 %res
}

; Test case for https://github.com/llvm/llvm-project/issues/122496.
define i32 @iv_ext_used_outside( ptr %dst) {
; VEC-LABEL: define i32 @iv_ext_used_outside(
; VEC-SAME: ptr [[DST:%.*]]) {
; VEC-NEXT:  [[ENTRY:.*:]]
; VEC-NEXT:    br label %[[VECTOR_PH:.*]]
; VEC:       [[VECTOR_PH]]:
; VEC-NEXT:    br label %[[VECTOR_BODY:.*]]
; VEC:       [[VECTOR_BODY]]:
; VEC-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16
; VEC-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[OFFSET_IDX]]
; VEC-NEXT:    store <2 x i32> zeroinitializer, ptr [[TMP1]], align 4
; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; VEC-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
; VEC-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; VEC:       [[MIDDLE_BLOCK]]:
; VEC-NEXT:    br label %[[SCALAR_PH:.*]]
; VEC:       [[SCALAR_PH]]:
; VEC-NEXT:    br label %[[LOOP:.*]]
; VEC:       [[LOOP]]:
; VEC-NEXT:    [[IV_1:%.*]] = phi i16 [ 128, %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
; VEC-NEXT:    [[IV_2:%.*]] = phi i32 [ 128, %[[SCALAR_PH]] ], [ [[IV_1_EXT:%.*]], %[[LOOP]] ]
; VEC-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[IV_1]]
; VEC-NEXT:    store i32 0, ptr [[GEP]], align 4
; VEC-NEXT:    [[IV_1_NEXT]] = add nuw nsw i16 [[IV_1]], 1
; VEC-NEXT:    [[IV_1_EXT]] = zext nneg i16 [[IV_1_NEXT]] to i32
; VEC-NEXT:    [[EC:%.*]] = icmp samesign ult i16 [[IV_1]], 128
; VEC-NEXT:    br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]], {{!llvm.loop ![0-9]+}}
; VEC:       [[EXIT]]:
; VEC-NEXT:    [[IV_1_EXT_LCSSA:%.*]] = phi i32 [ [[IV_1_EXT]], %[[LOOP]] ]
; VEC-NEXT:    [[IV_2_LCSSA:%.*]] = phi i32 [ [[IV_2]], %[[LOOP]] ]
; VEC-NEXT:    [[ADD:%.*]] = add i32 [[IV_1_EXT_LCSSA]], [[IV_2_LCSSA]]
; VEC-NEXT:    ret i32 [[IV_1_EXT_LCSSA]]
;
; INTERLEAVE-LABEL: define i32 @iv_ext_used_outside(
; INTERLEAVE-SAME: ptr [[DST:%.*]]) {
; INTERLEAVE-NEXT:  [[ENTRY:.*:]]
; INTERLEAVE-NEXT:    br label %[[VECTOR_PH:.*]]
; INTERLEAVE:       [[VECTOR_PH]]:
; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
; INTERLEAVE:       [[VECTOR_BODY]]:
; INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTERLEAVE-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16
; INTERLEAVE-NEXT:    [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], 1
; INTERLEAVE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[OFFSET_IDX]]
; INTERLEAVE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[TMP1]]
; INTERLEAVE-NEXT:    store i32 0, ptr [[TMP2]], align 4
; INTERLEAVE-NEXT:    store i32 0, ptr [[TMP3]], align 4
; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; INTERLEAVE-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
; INTERLEAVE-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[MIDDLE_BLOCK]]:
; INTERLEAVE-NEXT:    br label %[[SCALAR_PH:.*]]
; INTERLEAVE:       [[SCALAR_PH]]:
; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
; INTERLEAVE:       [[LOOP]]:
; INTERLEAVE-NEXT:    [[IV_1:%.*]] = phi i16 [ 128, %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
; INTERLEAVE-NEXT:    [[IV_2:%.*]] = phi i32 [ 128, %[[SCALAR_PH]] ], [ [[IV_1_EXT:%.*]], %[[LOOP]] ]
; INTERLEAVE-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[IV_1]]
; INTERLEAVE-NEXT:    store i32 0, ptr [[GEP]], align 4
; INTERLEAVE-NEXT:    [[IV_1_NEXT]] = add nuw nsw i16 [[IV_1]], 1
; INTERLEAVE-NEXT:    [[IV_1_EXT]] = zext nneg i16 [[IV_1_NEXT]] to i32
; INTERLEAVE-NEXT:    [[EC:%.*]] = icmp samesign ult i16 [[IV_1]], 128
; INTERLEAVE-NEXT:    br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[EXIT]]:
; INTERLEAVE-NEXT:    [[IV_1_EXT_LCSSA:%.*]] = phi i32 [ [[IV_1_EXT]], %[[LOOP]] ]
; INTERLEAVE-NEXT:    [[IV_2_LCSSA:%.*]] = phi i32 [ [[IV_2]], %[[LOOP]] ]
; INTERLEAVE-NEXT:    [[ADD:%.*]] = add i32 [[IV_1_EXT_LCSSA]], [[IV_2_LCSSA]]
; INTERLEAVE-NEXT:    ret i32 [[IV_1_EXT_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @iv_ext_used_outside(
; TAILFOLD-SAME: ptr [[DST:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
; TAILFOLD-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 [[INDEX]], i32 129)
; TAILFOLD-NEXT:    [[TMP0:%.*]] = trunc i32 [[INDEX]] to i16
; TAILFOLD-NEXT:    [[TMP1:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
; TAILFOLD-NEXT:    br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; TAILFOLD:       [[PRED_STORE_IF]]:
; TAILFOLD-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[TMP0]]
; TAILFOLD-NEXT:    store i32 0, ptr [[TMP2]], align 4
; TAILFOLD-NEXT:    br label %[[PRED_STORE_CONTINUE]]
; TAILFOLD:       [[PRED_STORE_CONTINUE]]:
; TAILFOLD-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
; TAILFOLD-NEXT:    br i1 [[TMP3]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; TAILFOLD:       [[PRED_STORE_IF1]]:
; TAILFOLD-NEXT:    [[TMP4:%.*]] = add i16 [[TMP0]], 1
; TAILFOLD-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[TMP4]]
; TAILFOLD-NEXT:    store i32 0, ptr [[TMP5]], align 4
; TAILFOLD-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
; TAILFOLD:       [[PRED_STORE_CONTINUE2]]:
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 130
; TAILFOLD-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    br label %[[EXIT:.*]]
; TAILFOLD:       [[EXIT]]:
; TAILFOLD-NEXT:    [[ADD:%.*]] = add i32 129, 128
; TAILFOLD-NEXT:    ret i32 129
;
entry:
  br label %loop

loop:
  %iv.1 = phi i16 [ 0, %entry ], [ %iv.1.next, %loop ]
  %iv.2 = phi i32 [ 0, %entry ], [ %iv.1.ext, %loop ]
  %gep = getelementptr inbounds nuw i32, ptr %dst, i16 %iv.1
  store i32 0, ptr %gep, align 4
  %iv.1.next = add nuw nsw i16 %iv.1, 1
  %iv.1.ext = zext nneg i16 %iv.1.next to i32
  %ec = icmp samesign ult i16 %iv.1, 128
  br i1 %ec, label %loop, label %exit

exit:
  %iv.1.ext.lcssa = phi i32 [ %iv.1.ext, %loop ]
  %iv.2.lcssa = phi i32 [ %iv.2, %loop ]
  %add = add i32 %iv.1.ext.lcssa, %iv.2.lcssa
  ret i32 %iv.1.ext.lcssa
}

; Test case for https://github.com/llvm/llvm-project/issues/122602.
define i64 @test_iv_increment_incremented(ptr %dst) {
; VEC-LABEL: define i64 @test_iv_increment_incremented(
; VEC-SAME: ptr [[DST:%.*]]) {
; VEC-NEXT:  [[ENTRY:.*:]]
; VEC-NEXT:    br label %[[VECTOR_PH:.*]]
; VEC:       [[VECTOR_PH]]:
; VEC-NEXT:    br label %[[VECTOR_BODY:.*]]
; VEC:       [[VECTOR_BODY]]:
; VEC-NEXT:    [[TMP0:%.*]] = getelementptr i16, ptr [[DST]], i64 3
; VEC-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr [[TMP0]], i64 -1
; VEC-NEXT:    store <2 x i16> splat (i16 1), ptr [[TMP2]], align 2
; VEC-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
; VEC:       [[MIDDLE_BLOCK]]:
; VEC-NEXT:    br label %[[EXIT:.*]]
; VEC:       [[EXIT]]:
; VEC-NEXT:    ret i64 1
;
; INTERLEAVE-LABEL: define i64 @test_iv_increment_incremented(
; INTERLEAVE-SAME: ptr [[DST:%.*]]) {
; INTERLEAVE-NEXT:  [[ENTRY:.*:]]
; INTERLEAVE-NEXT:    br label %[[VECTOR_PH:.*]]
; INTERLEAVE:       [[VECTOR_PH]]:
; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
; INTERLEAVE:       [[VECTOR_BODY]]:
; INTERLEAVE-NEXT:    [[TMP0:%.*]] = getelementptr i16, ptr [[DST]], i64 3
; INTERLEAVE-NEXT:    [[TMP1:%.*]] = getelementptr i16, ptr [[DST]], i64 2
; INTERLEAVE-NEXT:    store i16 1, ptr [[TMP0]], align 2
; INTERLEAVE-NEXT:    store i16 1, ptr [[TMP1]], align 2
; INTERLEAVE-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
; INTERLEAVE:       [[MIDDLE_BLOCK]]:
; INTERLEAVE-NEXT:    br label %[[EXIT:.*]]
; INTERLEAVE:       [[EXIT]]:
; INTERLEAVE-NEXT:    ret i64 1
;
; TAILFOLD-LABEL: define i64 @test_iv_increment_incremented(
; TAILFOLD-SAME: ptr [[DST:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[TMP0:%.*]] = getelementptr i16, ptr [[DST]], i64 3
; TAILFOLD-NEXT:    [[TMP1:%.*]] = getelementptr i16, ptr [[TMP0]], i64 -1
; TAILFOLD-NEXT:    store <2 x i16> splat (i16 1), ptr [[TMP1]], align 2
; TAILFOLD-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    br label %[[EXIT:.*]]
; TAILFOLD:       [[EXIT]]:
; TAILFOLD-NEXT:    ret i64 1
;
entry:
  br label %loop

loop:
  %iv.1 = phi i64 [ 3, %entry ], [ %iv.1.next, %loop ]
  %iv.2 = phi i64 [ 2, %entry ], [ %iv.2.next, %loop ]
  %gep = getelementptr i16, ptr %dst, i64 %iv.1
  store i16 1, ptr %gep, align 2
  %iv.2.next = add i64 %iv.2, -1
  %ec = icmp eq i64 %iv.2.next, 0
  %iv.1.next = add i64 %iv.2.next, 1
  br i1 %ec, label %exit, label %loop

exit:
  ret i64 %iv.1.next
}

define i32 @cast_incremented_iv_live_out(ptr %arr, i32 %n) {
; VEC-LABEL: define i32 @cast_incremented_iv_live_out(
; VEC-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
; VEC-NEXT:  [[ENTRY:.*]]:
; VEC-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
; VEC-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 1)
; VEC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 2
; VEC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VEC:       [[VECTOR_PH]]:
; VEC-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[UMAX]], 1
; VEC-NEXT:    [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
; VEC-NEXT:    br label %[[VECTOR_BODY:.*]]
; VEC:       [[VECTOR_BODY]]:
; VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[INDEX]]
; VEC-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP1]], align 1
; VEC-NEXT:    [[TMP2:%.*]] = add <2 x i8> [[WIDE_LOAD]], splat (i8 1)
; VEC-NEXT:    store <2 x i8> [[TMP2]], ptr [[TMP1]], align 1
; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VEC-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; VEC-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; VEC:       [[MIDDLE_BLOCK]]:
; VEC-NEXT:    [[TMP4:%.*]] = sub nuw i64 [[N_VEC]], 1
; VEC-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
; VEC-NEXT:    [[TMP6:%.*]] = add i32 1, [[TMP5]]
; VEC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
; VEC-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VEC:       [[SCALAR_PH]]:
; VEC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; VEC-NEXT:    br label %[[LOOP:.*]]
; VEC:       [[LOOP]]:
; VEC-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; VEC-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[IV]]
; VEC-NEXT:    [[VAL:%.*]] = load i8, ptr [[GEP]], align 1
; VEC-NEXT:    [[VAL_INC:%.*]] = add i8 [[VAL]], 1
; VEC-NEXT:    store i8 [[VAL_INC]], ptr [[GEP]], align 1
; VEC-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
; VEC-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV_NEXT]] to i32
; VEC-NEXT:    [[COND:%.*]] = icmp ult i32 [[IV_TRUNC]], [[N]]
; VEC-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT]], {{!llvm.loop ![0-9]+}}
; VEC:       [[EXIT]]:
; VEC-NEXT:    [[IV_TRUNC_LCSSA:%.*]] = phi i32 [ [[IV_TRUNC]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
; VEC-NEXT:    ret i32 [[IV_TRUNC_LCSSA]]
;
; INTERLEAVE-LABEL: define i32 @cast_incremented_iv_live_out(
; INTERLEAVE-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
; INTERLEAVE-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
; INTERLEAVE-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 1)
; INTERLEAVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 2
; INTERLEAVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTERLEAVE:       [[VECTOR_PH]]:
; INTERLEAVE-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[UMAX]], 1
; INTERLEAVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
; INTERLEAVE:       [[VECTOR_BODY]]:
; INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTERLEAVE-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
; INTERLEAVE-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[INDEX]]
; INTERLEAVE-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[TMP1]]
; INTERLEAVE-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1
; INTERLEAVE-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP3]], align 1
; INTERLEAVE-NEXT:    [[TMP6:%.*]] = add i8 [[TMP4]], 1
; INTERLEAVE-NEXT:    [[TMP7:%.*]] = add i8 [[TMP5]], 1
; INTERLEAVE-NEXT:    store i8 [[TMP6]], ptr [[TMP2]], align 1
; INTERLEAVE-NEXT:    store i8 [[TMP7]], ptr [[TMP3]], align 1
; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; INTERLEAVE-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; INTERLEAVE-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[MIDDLE_BLOCK]]:
; INTERLEAVE-NEXT:    [[TMP9:%.*]] = sub nuw i64 [[N_VEC]], 1
; INTERLEAVE-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
; INTERLEAVE-NEXT:    [[TMP11:%.*]] = add i32 1, [[TMP10]]
; INTERLEAVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
; INTERLEAVE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; INTERLEAVE:       [[SCALAR_PH]]:
; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
; INTERLEAVE:       [[LOOP]]:
; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; INTERLEAVE-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[IV]]
; INTERLEAVE-NEXT:    [[VAL:%.*]] = load i8, ptr [[GEP]], align 1
; INTERLEAVE-NEXT:    [[VAL_INC:%.*]] = add i8 [[VAL]], 1
; INTERLEAVE-NEXT:    store i8 [[VAL_INC]], ptr [[GEP]], align 1
; INTERLEAVE-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
; INTERLEAVE-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV_NEXT]] to i32
; INTERLEAVE-NEXT:    [[COND:%.*]] = icmp ult i32 [[IV_TRUNC]], [[N]]
; INTERLEAVE-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[EXIT]]:
; INTERLEAVE-NEXT:    [[IV_TRUNC_LCSSA:%.*]] = phi i32 [ [[IV_TRUNC]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT:    ret i32 [[IV_TRUNC_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @cast_incremented_iv_live_out(
; TAILFOLD-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
; TAILFOLD-NEXT:    [[TMP1:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP0]], i64 1)
; TAILFOLD-NEXT:    br label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP1]], 1
; TAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 1
; TAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
; TAILFOLD-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[TMP1]])
; TAILFOLD-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
; TAILFOLD-NEXT:    br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; TAILFOLD:       [[PRED_STORE_IF]]:
; TAILFOLD-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[INDEX]]
; TAILFOLD-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
; TAILFOLD-NEXT:    [[TMP5:%.*]] = add i8 [[TMP4]], 1
; TAILFOLD-NEXT:    store i8 [[TMP5]], ptr [[TMP3]], align 1
; TAILFOLD-NEXT:    br label %[[PRED_STORE_CONTINUE]]
; TAILFOLD:       [[PRED_STORE_CONTINUE]]:
; TAILFOLD-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
; TAILFOLD-NEXT:    br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; TAILFOLD:       [[PRED_STORE_IF1]]:
; TAILFOLD-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 1
; TAILFOLD-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[ARR]], i64 [[TMP7]]
; TAILFOLD-NEXT:    [[TMP9:%.*]] = load i8, ptr [[TMP8]], align 1
; TAILFOLD-NEXT:    [[TMP10:%.*]] = add i8 [[TMP9]], 1
; TAILFOLD-NEXT:    store i8 [[TMP10]], ptr [[TMP8]], align 1
; TAILFOLD-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
; TAILFOLD:       [[PRED_STORE_CONTINUE2]]:
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; TAILFOLD-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    [[TMP12:%.*]] = sub nuw i64 [[TMP1]], 1
; TAILFOLD-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i32
; TAILFOLD-NEXT:    [[TMP15:%.*]] = add i32 1, [[TMP13]]
; TAILFOLD-NEXT:    br label %[[EXIT:.*]]
; TAILFOLD:       [[EXIT]]:
; TAILFOLD-NEXT:    ret i32 [[TMP15]]
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %gep = getelementptr i8, ptr %arr, i64 %iv
  %val = load i8, ptr %gep, align 1
  %val.inc = add i8 %val, 1
  store i8 %val.inc, ptr %gep, align 1
  %iv.next = add i64 %iv, 1
  %iv.trunc = trunc i64 %iv.next to i32
  %cond = icmp ult i32 %iv.trunc, %n
  br i1 %cond, label %loop, label %exit

exit:
  ret i32 %iv.trunc
}

define i32 @added_step(i32 %n, i32 %step_base, ptr %p) {
; VEC-LABEL: define i32 @added_step(
; VEC-SAME: i32 [[N:%.*]], i32 [[STEP_BASE:%.*]], ptr [[P:%.*]]) {
; VEC-NEXT:  [[ENTRY:.*]]:
; VEC-NEXT:    [[STEP:%.*]] = add i32 [[STEP_BASE]], 1
; VEC-NEXT:    [[UMAX1:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)
; VEC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[UMAX1]], 2
; VEC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; VEC:       [[VECTOR_SCEVCHECK]]:
; VEC-NEXT:    [[TMP0:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[N]], i32 1)
; VEC-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0
; VEC-NEXT:    br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; VEC:       [[VECTOR_PH]]:
; VEC-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[UMAX1]], 1
; VEC-NEXT:    [[N_VEC:%.*]] = sub i32 [[UMAX1]], [[N_MOD_VF]]
; VEC-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[STEP]], i64 0
; VEC-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
; VEC-NEXT:    br label %[[VECTOR_BODY:.*]]
; VEC:       [[VECTOR_BODY]]:
; VEC-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VEC-NEXT:    [[TMP2:%.*]] = mul <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; VEC-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
; VEC-NEXT:    store <2 x i32> [[TMP2]], ptr [[TMP3]], align 4
; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; VEC-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
; VEC-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; VEC-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; VEC:       [[MIDDLE_BLOCK]]:
; VEC-NEXT:    [[TMP5:%.*]] = sub nuw i32 [[N_VEC]], 1
; VEC-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], [[STEP]]
; VEC-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[UMAX1]], [[N_VEC]]
; VEC-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VEC:       [[SCALAR_PH]]:
; VEC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; VEC-NEXT:    br label %[[LOOP:.*]]
; VEC:       [[LOOP]]:
; VEC-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; VEC-NEXT:    [[DERIVED:%.*]] = mul i32 [[IV]], [[STEP]]
; VEC-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
; VEC-NEXT:    store i32 [[DERIVED]], ptr [[GEP]], align 4
; VEC-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
; VEC-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IV_NEXT]], [[N]]
; VEC-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], {{!llvm.loop ![0-9]+}}
; VEC:       [[EXIT]]:
; VEC-NEXT:    [[DERIVED_LCSSA:%.*]] = phi i32 [ [[DERIVED]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
; VEC-NEXT:    ret i32 [[DERIVED_LCSSA]]
;
; INTERLEAVE-LABEL: define i32 @added_step(
; INTERLEAVE-SAME: i32 [[N:%.*]], i32 [[STEP_BASE:%.*]], ptr [[P:%.*]]) {
; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
; INTERLEAVE-NEXT:    [[STEP:%.*]] = add i32 [[STEP_BASE]], 1
; INTERLEAVE-NEXT:    [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)
; INTERLEAVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[UMAX]], 2
; INTERLEAVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; INTERLEAVE:       [[VECTOR_PH]]:
; INTERLEAVE-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[UMAX]], 1
; INTERLEAVE-NEXT:    [[N_VEC:%.*]] = sub i32 [[UMAX]], [[N_MOD_VF]]
; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
; INTERLEAVE:       [[VECTOR_BODY]]:
; INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; INTERLEAVE-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 1
; INTERLEAVE-NEXT:    [[TMP1:%.*]] = mul i32 [[INDEX]], [[STEP]]
; INTERLEAVE-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP0]], [[STEP]]
; INTERLEAVE-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
; INTERLEAVE-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP0]]
; INTERLEAVE-NEXT:    store i32 [[TMP1]], ptr [[TMP3]], align 4
; INTERLEAVE-NEXT:    store i32 [[TMP2]], ptr [[TMP4]], align 4
; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; INTERLEAVE-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; INTERLEAVE-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[MIDDLE_BLOCK]]:
; INTERLEAVE-NEXT:    [[TMP6:%.*]] = sub nuw i32 [[N_VEC]], 1
; INTERLEAVE-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], [[STEP]]
; INTERLEAVE-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[UMAX]], [[N_VEC]]
; INTERLEAVE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; INTERLEAVE:       [[SCALAR_PH]]:
; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
; INTERLEAVE:       [[LOOP]]:
; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; INTERLEAVE-NEXT:    [[DERIVED:%.*]] = mul i32 [[IV]], [[STEP]]
; INTERLEAVE-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
; INTERLEAVE-NEXT:    store i32 [[DERIVED]], ptr [[GEP]], align 4
; INTERLEAVE-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
; INTERLEAVE-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IV_NEXT]], [[N]]
; INTERLEAVE-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], {{!llvm.loop ![0-9]+}}
; INTERLEAVE:       [[EXIT]]:
; INTERLEAVE-NEXT:    [[DERIVED_LCSSA:%.*]] = phi i32 [ [[DERIVED]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
; INTERLEAVE-NEXT:    ret i32 [[DERIVED_LCSSA]]
;
; TAILFOLD-LABEL: define i32 @added_step(
; TAILFOLD-SAME: i32 [[N:%.*]], i32 [[STEP_BASE:%.*]], ptr [[P:%.*]]) {
; TAILFOLD-NEXT:  [[ENTRY:.*:]]
; TAILFOLD-NEXT:    [[STEP:%.*]] = add i32 [[STEP_BASE]], 1
; TAILFOLD-NEXT:    [[TMP0:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)
; TAILFOLD-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
; TAILFOLD:       [[VECTOR_SCEVCHECK]]:
; TAILFOLD-NEXT:    [[TMP1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[N]], i32 1)
; TAILFOLD-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0
; TAILFOLD-NEXT:    br i1 [[TMP2]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; TAILFOLD:       [[VECTOR_PH]]:
; TAILFOLD-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP0]], 1
; TAILFOLD-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 1
; TAILFOLD-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; TAILFOLD-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[STEP]], i64 0
; TAILFOLD-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
; TAILFOLD-NEXT:    br label %[[VECTOR_BODY:.*]]
; TAILFOLD:       [[VECTOR_BODY]]:
; TAILFOLD-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
; TAILFOLD-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
; TAILFOLD-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 [[INDEX]], i32 [[TMP0]])
; TAILFOLD-NEXT:    [[TMP3:%.*]] = mul <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; TAILFOLD-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
; TAILFOLD-NEXT:    br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; TAILFOLD:       [[PRED_STORE_IF]]:
; TAILFOLD-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
; TAILFOLD-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP3]], i64 0
; TAILFOLD-NEXT:    store i32 [[TMP6]], ptr [[TMP5]], align 4
; TAILFOLD-NEXT:    br label %[[PRED_STORE_CONTINUE]]
; TAILFOLD:       [[PRED_STORE_CONTINUE]]:
; TAILFOLD-NEXT:    [[TMP7:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
; TAILFOLD-NEXT:    br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; TAILFOLD:       [[PRED_STORE_IF1]]:
; TAILFOLD-NEXT:    [[TMP8:%.*]] = add i32 [[INDEX]], 1
; TAILFOLD-NEXT:    [[TMP9:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP8]]
; TAILFOLD-NEXT:    [[TMP10:%.*]] = extractelement <2 x i32> [[TMP3]], i64 1
; TAILFOLD-NEXT:    store i32 [[TMP10]], ptr [[TMP9]], align 4
; TAILFOLD-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
; TAILFOLD:       [[PRED_STORE_CONTINUE2]]:
; TAILFOLD-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
; TAILFOLD-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
; TAILFOLD-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; TAILFOLD-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[MIDDLE_BLOCK]]:
; TAILFOLD-NEXT:    [[TMP12:%.*]] = sub nuw i32 [[TMP0]], 1
; TAILFOLD-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP12]], [[STEP]]
; TAILFOLD-NEXT:    br label %[[EXIT:.*]]
; TAILFOLD:       [[SCALAR_PH]]:
; TAILFOLD-NEXT:    br label %[[LOOP:.*]]
; TAILFOLD:       [[LOOP]]:
; TAILFOLD-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; TAILFOLD-NEXT:    [[DERIVED:%.*]] = mul i32 [[IV]], [[STEP]]
; TAILFOLD-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
; TAILFOLD-NEXT:    store i32 [[DERIVED]], ptr [[GEP]], align 4
; TAILFOLD-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
; TAILFOLD-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IV_NEXT]], [[N]]
; TAILFOLD-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], {{!llvm.loop ![0-9]+}}
; TAILFOLD:       [[EXIT]]:
; TAILFOLD-NEXT:    [[DERIVED_LCSSA:%.*]] = phi i32 [ [[DERIVED]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
; TAILFOLD-NEXT:    ret i32 [[DERIVED_LCSSA]]
;
entry:
  %step = add i32 %step_base, 1
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
  %derived = mul i32 %iv, %step
  %gep = getelementptr i32, ptr %p, i32 %iv
  store i32 %derived, ptr %gep, align 4
  %iv.next = add i32 %iv, 1
  %cmp = icmp ult i32 %iv.next, %n
  br i1 %cmp, label %loop, label %exit

exit:
  ret i32 %derived
}

define i64 @zext_iv_outside_user() {
; COMMON-LABEL: define i64 @zext_iv_outside_user() {
; COMMON-NEXT:  [[ENTRY:.*:]]
; COMMON-NEXT:    br label %[[VECTOR_PH:.*]]
; COMMON:       [[VECTOR_PH]]:
; COMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMMON:       [[VECTOR_BODY]]:
; COMMON-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMMON-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; COMMON-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], -2147483644
; COMMON-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
; COMMON:       [[MIDDLE_BLOCK]]:
; COMMON-NEXT:    br label %[[EXIT:.*]]
; COMMON:       [[EXIT]]:
; COMMON-NEXT:    ret i64 2147483651
;
entry:
  br label %loop

loop:
  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
  %wide = zext i32 %i to i64
  %i.next = add i32 %i, 1
  %c = icmp ult i32 %i.next, 2147483652
  br i1 %c, label %loop, label %exit

exit:
  %r = phi i64 [ %wide, %loop ]
  ret i64 %r
}