; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s --check-prefixes=CHECK,DEFAULT
; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -lv-strided-pointer-ivs -S %s | FileCheck %s --check-prefixes=CHECK,STRIDED

; Negative test: the pointer is advanced inside the inner loop (*p++), so the
; outer-loop phi is not an AddRec and is not an induction.
define void @ptr_iv_incremented_in_inner(ptr noalias %dst, i64 %n) {
; CHECK-LABEL: define void @ptr_iv_incremented_in_inner(
; CHECK-SAME: ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OUTER_PTR:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[INNER_PTR_NEXT_LCSSA:%.*]], %[[OUTER_LATCH]] ]
; CHECK-NEXT:    br label %[[INNER:.*]]
; CHECK:       [[INNER]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER]] ]
; CHECK-NEXT:    [[INNER_PTR:%.*]] = phi ptr [ [[OUTER_PTR]], %[[OUTER_HEADER]] ], [ [[INNER_PTR_NEXT:%.*]], %[[INNER]] ]
; CHECK-NEXT:    store i64 [[OUTER_IV]], ptr [[INNER_PTR]], align 8
; CHECK-NEXT:    [[INNER_PTR_NEXT]] = getelementptr inbounds i64, ptr [[INNER_PTR]], i64 1
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_EC:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 8
; CHECK-NEXT:    br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[INNER_PTR_NEXT_LCSSA]] = phi ptr [ [[INNER_PTR_NEXT]], %[[INNER]] ]
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_EC]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.ptr = phi ptr [ %dst, %entry ], [ %inner.ptr.next.lcssa, %outer.latch ]
  br label %inner

inner:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
  %inner.ptr = phi ptr [ %outer.ptr, %outer.header ], [ %inner.ptr.next, %inner ]
  store i64 %outer.iv, ptr %inner.ptr, align 8
  %inner.ptr.next = getelementptr inbounds i64, ptr %inner.ptr, i64 1
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.ec = icmp eq i64 %inner.iv.next, 8
  br i1 %inner.ec, label %outer.latch, label %inner

outer.latch:
  %inner.ptr.next.lcssa = phi ptr [ %inner.ptr.next, %inner ]
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.ec = icmp eq i64 %outer.iv.next, %n
  br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; Negative test: floating-point inductions are not supported in the outer-loop
; path.
define void @fp_iv(ptr noalias %dst, i64 %n) {
; CHECK-LABEL: define void @fp_iv(
; CHECK-SAME: ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[FP_IV:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[FP_IV_NEXT:%.*]], %[[OUTER_LATCH]] ]
; CHECK-NEXT:    br label %[[INNER:.*]]
; CHECK:       [[INNER]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER]] ]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_EC:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 8
; CHECK-NEXT:    br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[OUTER_IV]]
; CHECK-NEXT:    store float [[FP_IV]], ptr [[GEP_DST]], align 4
; CHECK-NEXT:    [[FP_IV_NEXT]] = fadd float [[FP_IV]], 1.000000e+00
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_EC]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP0]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %fp.iv = phi float [ 0.0, %entry ], [ %fp.iv.next, %outer.latch ]
  br label %inner

inner:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.ec = icmp eq i64 %inner.iv.next, 8
  br i1 %inner.ec, label %outer.latch, label %inner

outer.latch:
  %gep.dst = getelementptr inbounds float, ptr %dst, i64 %outer.iv
  store float %fp.iv, ptr %gep.dst, align 4
  %fp.iv.next = fadd float %fp.iv, 1.0
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.ec = icmp eq i64 %outer.iv.next, %n
  br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The pointer IV has a unit step, so the store through it is consecutive.
define void @ptr_iv_unit_step(ptr noalias %dst, i64 %n) {
; CHECK-LABEL: define void @ptr_iv_unit_step(
; CHECK-SAME: ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[N_VEC]], 3
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH3:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH3]] ]
; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[INDEX]], 3
; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
; CHECK-NEXT:    br label %[[INNER1:.*]]
; CHECK:       [[INNER1]]:
; CHECK-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER1]] ]
; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 8)
; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
; CHECK-NEXT:    br i1 [[TMP6]], label %[[OUTER_LATCH3]], label %[[INNER1]]
; CHECK:       [[OUTER_LATCH3]]:
; CHECK-NEXT:    store <4 x i64> [[VEC_IND]], ptr [[NEXT_GEP]], align 8
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %outer.latch ]
  br label %inner

inner:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.ec = icmp eq i64 %inner.iv.next, 8
  br i1 %inner.ec, label %outer.latch, label %inner

outer.latch:
  store i64 %outer.iv, ptr %ptr.iv, align 8
  %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.ec = icmp eq i64 %outer.iv.next, %n
  br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The pointer IV has a constant non-unit step.
define void @ptr_iv_non_unit_step(ptr noalias %dst, i64 %n) {
; CHECK-LABEL: define void @ptr_iv_non_unit_step(
; CHECK-SAME: ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[N_VEC]], 24
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH3:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH3]] ]
; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[DST]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[OUTER_LATCH3]] ]
; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 24, i64 48, i64 72>
; CHECK-NEXT:    br label %[[INNER1:.*]]
; CHECK:       [[INNER1]]:
; CHECK-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER1]] ]
; CHECK-NEXT:    [[TMP3]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <4 x i64> [[TMP3]], splat (i64 8)
; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0
; CHECK-NEXT:    br i1 [[TMP5]], label %[[OUTER_LATCH3]], label %[[INNER1]]
; CHECK:       [[OUTER_LATCH3]]:
; CHECK-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[VEC_IND]], <4 x ptr> align 8 [[VECTOR_GEP]], <4 x i1> splat (i1 true))
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 96
; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %outer.latch ]
  br label %inner

inner:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.ec = icmp eq i64 %inner.iv.next, 8
  br i1 %inner.ec, label %outer.latch, label %inner

outer.latch:
  store i64 %outer.iv, ptr %ptr.iv, align 8
  %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 3
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.ec = icmp eq i64 %outer.iv.next, %n
  br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The pointer IV is used inside the inner loop, and the value of the pointer IV
; phi in the last iteration is used after the loop.
define ptr @ptr_iv_used_in_inner_and_live_out(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK-LABEL: define ptr @ptr_iv_used_in_inner_and_live_out(
; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[N_VEC]], 6
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]]
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ]
; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[SRC]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[OUTER_LATCH4]] ]
; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 64, i64 128, i64 192>
; CHECK-NEXT:    br label %[[INNER1:.*]]
; CHECK:       [[INNER1]]:
; CHECK-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER1]] ]
; CHECK-NEXT:    [[SUM3:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER1]] ]
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds i64, <4 x ptr> [[VECTOR_GEP]], <4 x i64> [[INNER_IV2]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
; CHECK-NEXT:    [[TMP3]] = add <4 x i64> [[SUM3]], [[WIDE_MASKED_GATHER]]
; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 8)
; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
; CHECK-NEXT:    br i1 [[TMP6]], label %[[OUTER_LATCH4]], label %[[INNER1]]
; CHECK:       [[OUTER_LATCH4]]:
; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT:    store <4 x i64> [[TMP3]], ptr [[TMP7]], align 8
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 256
; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -64
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %ptr.iv = phi ptr [ %src, %entry ], [ %ptr.iv.next, %outer.latch ]
  br label %inner

inner:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
  %sum = phi i64 [ 0, %outer.header ], [ %sum.next, %inner ]
  %gep.src = getelementptr inbounds i64, ptr %ptr.iv, i64 %inner.iv
  %l = load i64, ptr %gep.src, align 8
  %sum.next = add i64 %sum, %l
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.ec = icmp eq i64 %inner.iv.next, 8
  br i1 %inner.ec, label %outer.latch, label %inner

outer.latch:
  %gep.dst = getelementptr inbounds i64, ptr %dst, i64 %outer.iv
  store i64 %sum.next, ptr %gep.dst, align 8
  %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 8
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.ec = icmp eq i64 %outer.iv.next, %n
  br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !0

exit:
  %ptr.iv.lcssa = phi ptr [ %ptr.iv, %outer.latch ]
  ret ptr %ptr.iv.lcssa
}

; The pointer IV is the only outer-loop induction and controls the exit.
define void @ptr_iv_only(ptr noalias %dst, ptr %end) {
; CHECK-LABEL: define void @ptr_iv_only(
; CHECK-SAME: ptr noalias [[DST:%.*]], ptr [[END:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[END2:%.*]] = ptrtoaddr ptr [[END]] to i64
; CHECK-NEXT:    [[DST1:%.*]] = ptrtoaddr ptr [[DST]] to i64
; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[END2]], -8
; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[DST1]]
; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP4:%.*]] = and i64 [[TMP3]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[TMP4]]
; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[N_VEC]], 3
; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH5:.*]] ]
; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[INDEX]], 3
; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
; CHECK-NEXT:    br label %[[INNER3:.*]]
; CHECK:       [[INNER3]]:
; CHECK-NEXT:    [[INNER_IV4:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP8:%.*]], %[[INNER3]] ]
; CHECK-NEXT:    [[TMP8]] = add nuw nsw <4 x i64> [[INNER_IV4]], splat (i64 1)
; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <4 x i64> [[TMP8]], splat (i64 8)
; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP9]], i64 0
; CHECK-NEXT:    br i1 [[TMP10]], label %[[OUTER_LATCH5]], label %[[INNER3]]
; CHECK:       [[OUTER_LATCH5]]:
; CHECK-NEXT:    store <4 x i64> zeroinitializer, ptr [[NEXT_GEP]], align 8
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %outer.latch ]
  br label %inner

inner:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.ec = icmp eq i64 %inner.iv.next, 8
  br i1 %inner.ec, label %outer.latch, label %inner

outer.latch:
  store i64 0, ptr %ptr.iv, align 8
  %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
  %outer.ec = icmp eq ptr %ptr.iv.next, %end
  br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The pointer IV step is a runtime value. As in the inner-loop path, this is only
; vectorized with -lv-strided-pointer-ivs.
define void @ptr_iv_runtime_step(ptr noalias %dst, i64 %n, i64 %step) {
; DEFAULT-LABEL: define void @ptr_iv_runtime_step(
; DEFAULT-SAME: ptr noalias [[DST:%.*]], i64 [[N:%.*]], i64 [[STEP:%.*]]) {
; DEFAULT-NEXT:  [[ENTRY:.*]]:
; DEFAULT-NEXT:    br label %[[OUTER_HEADER:.*]]
; DEFAULT:       [[OUTER_HEADER]]:
; DEFAULT-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; DEFAULT-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[OUTER_LATCH]] ]
; DEFAULT-NEXT:    br label %[[INNER:.*]]
; DEFAULT:       [[INNER]]:
; DEFAULT-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER]] ]
; DEFAULT-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; DEFAULT-NEXT:    [[INNER_EC:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 8
; DEFAULT-NEXT:    br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER]]
; DEFAULT:       [[OUTER_LATCH]]:
; DEFAULT-NEXT:    store i64 [[OUTER_IV]], ptr [[PTR_IV]], align 8
; DEFAULT-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i8, ptr [[PTR_IV]], i64 [[STEP]]
; DEFAULT-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; DEFAULT-NEXT:    [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; DEFAULT-NEXT:    br i1 [[OUTER_EC]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP0]]
; DEFAULT:       [[EXIT]]:
; DEFAULT-NEXT:    ret void
;
; STRIDED-LABEL: define void @ptr_iv_runtime_step(
; STRIDED-SAME: ptr noalias [[DST:%.*]], i64 [[N:%.*]], i64 [[STEP:%.*]]) {
; STRIDED-NEXT:  [[ENTRY:.*:]]
; STRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; STRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; STRIDED:       [[VECTOR_PH]]:
; STRIDED-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; STRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; STRIDED-NEXT:    [[TMP1:%.*]] = mul i64 [[N_VEC]], [[STEP]]
; STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; STRIDED-NEXT:    br label %[[VECTOR_BODY:.*]]
; STRIDED:       [[VECTOR_BODY]]:
; STRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH3:.*]] ]
; STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH3]] ]
; STRIDED-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[DST]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[OUTER_LATCH3]] ]
; STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STEP]], i64 0
; STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; STRIDED-NEXT:    [[TMP3:%.*]] = mul <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[BROADCAST_SPLAT]]
; STRIDED-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> [[TMP3]]
; STRIDED-NEXT:    br label %[[INNER1:.*]]
; STRIDED:       [[INNER1]]:
; STRIDED-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER1]] ]
; STRIDED-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
; STRIDED-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 8)
; STRIDED-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
; STRIDED-NEXT:    br i1 [[TMP6]], label %[[OUTER_LATCH3]], label %[[INNER1]]
; STRIDED:       [[OUTER_LATCH3]]:
; STRIDED-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[VEC_IND]], <4 x ptr> align 8 [[VECTOR_GEP]], <4 x i1> splat (i1 true))
; STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; STRIDED-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; STRIDED-NEXT:    [[TMP7:%.*]] = shl i64 [[STEP]], 2
; STRIDED-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP7]]
; STRIDED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; STRIDED-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; STRIDED:       [[MIDDLE_BLOCK]]:
; STRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; STRIDED-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; STRIDED:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %outer.latch ]
  br label %inner

inner:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.ec = icmp eq i64 %inner.iv.next, 8
  br i1 %inner.ec, label %outer.latch, label %inner

outer.latch:
  store i64 %outer.iv, ptr %ptr.iv, align 8
  %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 %step
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.ec = icmp eq i64 %outer.iv.next, %n
  br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.vectorize.width", i32 4}
!2 = !{!"llvm.loop.vectorize.enable"}