llvm/test/Transforms/LoopVectorize/induction-cost.ll

; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "(Cost of .*(WIDEN-INDUCTION|SCALAR-STEPS|DERIVED-IV|WIDEN-POINTER-INDUCTION))|(canonical IV increment)|(Cost for VF)" --filter-out-after "LV: (Selecting|Using user) VF" --version 6
; REQUIRES: asserts
; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck --check-prefix=VF2 %s
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck --check-prefix=VF4 %s

define void @int_induction(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'int_induction'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost for VF 2: 6 (Estimated cost per lane: 3)
;
; VF4-LABEL: 'int_induction'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost for VF 4: 6 (Estimated cost per lane: 1.5)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %gep = getelementptr inbounds i64, ptr %dst, i64 %iv
  store i64 %iv, ptr %gep, align 8
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

define void @fp_induction(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction'
; VF2:  Cost of 2 for VF 2: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 1 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF2:  Cost for VF 2: 6 (Estimated cost per lane: 3)
;
; VF4-LABEL: 'fp_induction'
; VF4:  Cost of 2 for VF 4: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 1 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF4:  Cost for VF 4: 6 (Estimated cost per lane: 1.5)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ]
  %gep = getelementptr inbounds float, ptr %dst, i64 %iv
  store float %fp.iv, ptr %gep, align 4
  %fp.iv.next = fadd fast float %fp.iv, 1.000000e+00
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

define void @trunc_induction(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'trunc_induction'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> (truncated to i32)
; VF2:  Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost for VF 2: 6 (Estimated cost per lane: 3)
;
; VF4-LABEL: 'trunc_induction'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> (truncated to i32)
; VF4:  Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost for VF 4: 6 (Estimated cost per lane: 1.5)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %iv.trunc = trunc i64 %iv to i32
  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
  store i32 %iv.trunc, ptr %gep, align 4
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

define void @trunc_and_wide_induction(ptr noalias %dst, ptr noalias %dst2, i64 %n) {
; VF2-LABEL: 'trunc_and_wide_induction'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 2 for VF 2: ir<%iv>.1 = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]> (truncated to i32)
; VF2:  Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost for VF 2: 9 (Estimated cost per lane: 4.5)
;
; VF4-LABEL: 'trunc_and_wide_induction'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 2 for VF 4: ir<%iv>.1 = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]> (truncated to i32)
; VF4:  Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost for VF 4: 9 (Estimated cost per lane: 2.25)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %iv.trunc = trunc i64 %iv to i32
  %gep = getelementptr inbounds i32, ptr %dst, i64 %iv
  store i32 %iv.trunc, ptr %gep, align 4
  %gep2 = getelementptr inbounds i64, ptr %dst2, i64 %iv
  store i64 %iv, ptr %gep2, align 8
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; All users of the FP induction need scalars, so no wide induction is generated
; and the scalar steps materialize one FAdd per lane, except for the first.
define void @fp_induction_scalar_users(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_scalar_users'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 1 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF2:  Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 1 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF2:  Cost for VF 2: 10 (Estimated cost per lane: 5)
;
; VF4-LABEL: 'fp_induction_scalar_users'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 1 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF4:  Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 1 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF4:  Cost for VF 4: 14 (Estimated cost per lane: 3.5)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ]
  %idx = mul i64 %iv, 3
  %gep = getelementptr inbounds float, ptr %dst, i64 %idx
  store float %fp.iv, ptr %gep, align 4
  %fp.iv.next = fadd fast float %fp.iv, 1.000000e+00
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; Same as @fp_induction_scalar_users, but for a decreasing induction, which uses
; FSub to compute the per-lane values.
define void @fp_induction_fsub_scalar_users(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_fsub_scalar_users'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 2 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<1.000000e+02> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF2:  Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 2 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<1.000000e+02> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF2:  Cost for VF 2: 11 (Estimated cost per lane: 5.5)
;
; VF4-LABEL: 'fp_induction_fsub_scalar_users'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 2 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<1.000000e+02> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF4:  Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 2 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<1.000000e+02> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF4:  Cost for VF 4: 15 (Estimated cost per lane: 3.75)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ 1.000000e+02, %entry ], [ %fp.iv.next, %loop ]
  %idx = mul i64 %iv, 3
  %gep = getelementptr inbounds float, ptr %dst, i64 %idx
  store float %fp.iv, ptr %gep, align 4
  %fp.iv.next = fsub fast float %fp.iv, 1.000000e+00
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; Same as @fp_induction_scalar_users, but with a loop-invariant step.
define void @fp_induction_invariant_step_scalar_users(ptr noalias %dst, float %step, i64 %n) {
; VF2-LABEL: 'fp_induction_invariant_step_scalar_users'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 3 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<%step>
; VF2:  Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<%step>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 3 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<%step>
; VF2:  Cost for VF 2: 12 (Estimated cost per lane: 6)
;
; VF4-LABEL: 'fp_induction_invariant_step_scalar_users'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 3 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<%step>
; VF4:  Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<%step>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 3 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<%step>
; VF4:  Cost for VF 4: 16 (Estimated cost per lane: 4)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ]
  %idx = mul i64 %iv, 3
  %gep = getelementptr inbounds float, ptr %dst, i64 %idx
  store float %fp.iv, ptr %gep, align 4
  %fp.iv.next = fadd fast float %fp.iv, %step
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; Same as @fp_induction_scalar_users (fadd, unit step, zero start), but
; without fast-math flags. The add is still expected to fold away: with a
; unit step the DERIVED-IV's addend is exactly sitofp(<the non-negative
; canonical/scalar IV>), which can never be -0.0, so "fadd 0.0, X" folds to X
; regardless of fast-math flags.
define void @fp_induction_no_fastmath_unit_step_zero_start(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_no_fastmath_unit_step_zero_start'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 1 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF2:  Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 1 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF2:  Cost for VF 2: 10 (Estimated cost per lane: 5)
;
; VF4-LABEL: 'fp_induction_no_fastmath_unit_step_zero_start'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 1 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF4:  Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 1 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF4:  Cost for VF 4: 14 (Estimated cost per lane: 3.5)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ]
  %idx = mul i64 %iv, 3
  %gep = getelementptr inbounds float, ptr %dst, i64 %idx
  store float %fp.iv, ptr %gep, align 4
  %fp.iv.next = fadd float %fp.iv, 1.000000e+00
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; Same as @fp_induction_no_fastmath_unit_step_zero_start, but with a
; loop-invariant, sign-unknown step. Unlike the unit-step case, the add is
; not known to fold away here: with a non-unit step the DERIVED-IV's addend
; is Step * sitofp(Index), whose sign depends on the unknown sign of Step, so
; it may be -0.0, and "fadd 0.0, X" doesn't fold to X in that case.
define void @fp_induction_no_fastmath_zero_start_invariant_step(ptr noalias %dst, float %step, i64 %n) {
; VF2-LABEL: 'fp_induction_no_fastmath_zero_start_invariant_step'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 3 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<%step>
; VF2:  Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<%step>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 3 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<%step>
; VF2:  Cost for VF 2: 12 (Estimated cost per lane: 6)
;
; VF4-LABEL: 'fp_induction_no_fastmath_zero_start_invariant_step'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 3 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<%step>
; VF4:  Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<%step>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 3 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<%step>
; VF4:  Cost for VF 4: 16 (Estimated cost per lane: 4)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ]
  %idx = mul i64 %iv, 3
  %gep = getelementptr inbounds float, ptr %dst, i64 %idx
  store float %fp.iv, ptr %gep, align 4
  %fp.iv.next = fadd float %fp.iv, %step
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; FSub never folds away for a zero start, regardless of the step: "fsub 0.0,
; X" isn't X, at best it simplifies to "fneg X", which is still a real
; instruction.
define void @fp_induction_fsub_zero_start(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_fsub_zero_start'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 2 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF2:  Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 2 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF2:  Cost for VF 2: 11 (Estimated cost per lane: 5.5)
;
; VF4-LABEL: 'fp_induction_fsub_zero_start'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 2 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF4:  Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 2 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF4:  Cost for VF 4: 15 (Estimated cost per lane: 3.75)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ]
  %idx = mul i64 %iv, 3
  %gep = getelementptr inbounds float, ptr %dst, i64 %idx
  store float %fp.iv, ptr %gep, align 4
  %fp.iv.next = fsub fast float %fp.iv, 1.000000e+00
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; Start of -0.0 so the FAdd can be folded.
define void @fp_induction_neg_zero_start_neg_step(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_neg_zero_start_neg_step'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 2 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<-0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<-2.000000e+00>
; VF2:  Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<-2.000000e+00>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 2 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<-0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<-2.000000e+00>
; VF2:  Cost for VF 2: 11 (Estimated cost per lane: 5.5)
;
; VF4-LABEL: 'fp_induction_neg_zero_start_neg_step'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 2 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<-0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<-2.000000e+00>
; VF4:  Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<-2.000000e+00>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 2 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<-0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<-2.000000e+00>
; VF4:  Cost for VF 4: 15 (Estimated cost per lane: 3.75)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ -0.000000e+00, %entry ], [ %fp.iv.next, %loop ]
  %idx = mul i64 %iv, 3
  %gep = getelementptr inbounds float, ptr %dst, i64 %idx
  store float %fp.iv, ptr %gep, align 4
  %fp.iv.next = fadd float %fp.iv, -2.000000e+00
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; Same as @fp_induction_neg_zero_start_neg_step, but with a +0.0 start: the
; FAdd cannot be folded.
define void @fp_induction_pos_zero_start_neg_step(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_pos_zero_start_neg_step'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 3 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<-2.000000e+00>
; VF2:  Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<-2.000000e+00>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 3 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<-2.000000e+00>
; VF2:  Cost for VF 2: 12 (Estimated cost per lane: 6)
;
; VF4-LABEL: 'fp_induction_pos_zero_start_neg_step'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 3 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<-2.000000e+00>
; VF4:  Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<-2.000000e+00>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 3 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<-2.000000e+00>
; VF4:  Cost for VF 4: 16 (Estimated cost per lane: 4)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ]
  %idx = mul i64 %iv, 3
  %gep = getelementptr inbounds float, ptr %dst, i64 %idx
  store float %fp.iv, ptr %gep, align 4
  %fp.iv.next = fadd float %fp.iv, -2.000000e+00
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; Same as @fp_induction_pos_zero_start_neg_step, but with a positive step.
define void @fp_induction_pos_zero_start_pos_step(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_pos_zero_start_pos_step'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 3 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<2.000000e+00>
; VF2:  Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<2.000000e+00>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 3 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<2.000000e+00>
; VF2:  Cost for VF 2: 12 (Estimated cost per lane: 6)
;
; VF4-LABEL: 'fp_induction_pos_zero_start_pos_step'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 3 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<2.000000e+00>
; VF4:  Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<2.000000e+00>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 3 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<2.000000e+00>
; VF4:  Cost for VF 4: 16 (Estimated cost per lane: 4)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ]
  %idx = mul i64 %iv, 3
  %gep = getelementptr inbounds float, ptr %dst, i64 %idx
  store float %fp.iv, ptr %gep, align 4
  %fp.iv.next = fadd float %fp.iv, 2.000000e+00
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; A non-zero start value never folds, so the FAdd is charged even though the
; unit step folds the FMul away.
define void @fp_induction_invariant_start(ptr noalias %dst, float %start, i64 %n) {
; VF2-LABEL: 'fp_induction_invariant_start'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 2 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF2:  Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 2 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF2:  Cost for VF 2: 11 (Estimated cost per lane: 5.5)
;
; VF4-LABEL: 'fp_induction_invariant_start'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 2 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00>
; VF4:  Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 2 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF4:  Cost for VF 4: 15 (Estimated cost per lane: 3.75)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ %start, %entry ], [ %fp.iv.next, %loop ]
  %idx = mul i64 %iv, 3
  %gep = getelementptr inbounds float, ptr %dst, i64 %idx
  store float %fp.iv, ptr %gep, align 4
  %fp.iv.next = fadd float %fp.iv, 1.000000e+00
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; The FP induction is only used outside the loop, so it is dead in the vector
; loop and nothing is charged for it.
define float @fp_induction_used_outside_loop_only(ptr noalias %dst, i64 %n) {
; VF2-LABEL: 'fp_induction_used_outside_loop_only'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost for VF 2: 8 (Estimated cost per lane: 4)
;
; VF4-LABEL: 'fp_induction_used_outside_loop_only'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost for VF 4: 8 (Estimated cost per lane: 2)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ]
  %gep = getelementptr inbounds i64, ptr %dst, i64 %iv
  store i64 %iv, ptr %gep, align 8
  %fp.iv.next = fadd fast float %fp.iv, 1.000000e+00
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret float %fp.iv.next
}

; The FP induction has both wide and scalar users, so both the wide induction
; and the scalar steps are generated and need to be accounted for.
define void @fp_induction_wide_and_scalar_users(ptr noalias %dst, ptr noalias %dst2, i64 %n) {
; VF2-LABEL: 'fp_induction_wide_and_scalar_users'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 2 for VF 2: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0]]>
; VF2:  Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4]]> * ir<1.000000e+00>
; VF2:  Cost of 1 for VF 2: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 1 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF2:  Cost for VF 2: 13 (Estimated cost per lane: 6.5)
;
; VF4-LABEL: 'fp_induction_wide_and_scalar_users'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 2 for VF 4: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0]]>
; VF4:  Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4]]> * ir<1.000000e+00>
; VF4:  Cost of 3 for VF 4: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 1 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF4:  Cost for VF 4: 17 (Estimated cost per lane: 4.25)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ]
  %gep = getelementptr inbounds float, ptr %dst, i64 %iv
  store float %fp.iv, ptr %gep, align 4
  %idx = mul i64 %iv, 3
  %gep2 = getelementptr inbounds float, ptr %dst2, i64 %idx
  store float %fp.iv, ptr %gep2, align 4
  %fp.iv.next = fadd fast float %fp.iv, 1.000000e+00
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; The scalar steps are inside a replicate region, for which no cost is computed
; yet.
define void @fp_induction_predicated(ptr noalias %dst, ptr noalias %cond, i64 %n) {
; VF2-LABEL: 'fp_induction_predicated'
; VF2:  Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4]]> * ir<1.000000e+00>
; VF2:  Cost of 0.5 for VF 2: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 1 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF2:  Cost for VF 2: 12.5 (Estimated cost per lane: 6)
;
; VF4-LABEL: 'fp_induction_predicated'
; VF4:  Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4]]> * ir<1.000000e+00>
; VF4:  Cost of 1.5 for VF 4: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1.000000e+00>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 1 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF4:  Cost for VF 4: 16.5 (Estimated cost per lane: 4)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
  %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %latch ]
  %c.gep = getelementptr inbounds i8, ptr %cond, i64 %iv
  %c = load i8, ptr %c.gep, align 1
  %c.0 = icmp eq i8 %c, 0
  br i1 %c.0, label %latch, label %then

then:
  %idx = mul i64 %iv, 3
  %gep = getelementptr inbounds float, ptr %dst, i64 %idx
  store float %fp.iv, ptr %gep, align 4
  br label %latch

latch:
  %fp.iv.next = fadd fast float %fp.iv, 1.000000e+00
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; The vector loop is executed exactly once for VF 4, so the induction increments
; and the canonical IV increment are all simplified away.
define void @fp_induction_tc_eq_4(ptr noalias %dst) {
; VF2-LABEL: 'fp_induction_tc_eq_4'
; VF2:  Cost of 2 for VF 2: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 1 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF2:  Cost for VF 2: 6 (Estimated cost per lane: 3)
;
; VF4-LABEL: 'fp_induction_tc_eq_4'
; VF4:  Cost of 0 for VF 4: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00>
; VF4:  Cost for VF 4: 2 (Estimated cost per lane: 0.5)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ]
  %gep = getelementptr inbounds float, ptr %dst, i64 %iv
  store float %fp.iv, ptr %gep, align 4
  %fp.iv.next = fadd fast float %fp.iv, 1.000000e+00
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 4
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; The loop is advanced by both an FP and a pointer induction. Neither induction
; is costed by the legacy cost model any more; the scalarized pointer induction
; is covered by its DERIVED-IV and SCALAR-STEPS recipes.
define void @fp_and_ptr_induction(ptr noalias %dst, ptr %end) {
; VF2-LABEL: 'fp_and_ptr_induction'
; VF2:  Cost of 2 for VF 2: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 1 for VF 2: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6:%[0-9]+]]> * ir<4>
; VF2:  Cost of 0 for VF 2: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<4>, vp<[[VP0]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<4>
; VF2:  Cost of 1 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2]]> * ir<1.000000e+00>
; VF2:  Cost for VF 2: 7 (Estimated cost per lane: 3.5)
;
; VF4-LABEL: 'fp_and_ptr_induction'
; VF4:  Cost of 2 for VF 4: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 1 for VF 4: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6:%[0-9]+]]> * ir<4>
; VF4:  Cost of 0 for VF 4: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<4>, vp<[[VP0]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<4>
; VF4:  Cost of 1 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2]]> * ir<1.000000e+00>
; VF4:  Cost for VF 4: 7 (Estimated cost per lane: 1.75)
;
entry:
  br label %loop

loop:
  %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
  %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ]
  store float %fp.iv, ptr %ptr.iv, align 4
  %fp.iv.next = fadd fast float %fp.iv, 1.000000e+00
  %ptr.iv.next = getelementptr inbounds float, ptr %ptr.iv, i64 1
  %ec = icmp eq ptr %ptr.iv.next, %end
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; Same as @fp_and_ptr_induction, but the loop is only advanced by a pointer
; induction. The canonical IV increment is still charged, even though the GEP
; incrementing the pointer induction is free in the scalar cost.
define void @ptr_induction_only(ptr %start, ptr %end) {
; VF2-LABEL: 'ptr_induction_only'
; VF2:  Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5:%[0-9]+]]>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP2:%[0-9]+]]> * ir<1>
; VF2:  Cost for VF 2: 6 (Estimated cost per lane: 3)
;
; VF4-LABEL: 'ptr_induction_only'
; VF4:  Cost of 0 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5:%[0-9]+]]>, ir<1>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP2:%[0-9]+]]> * ir<1>
; VF4:  Cost for VF 4: 6 (Estimated cost per lane: 1.5)
;
entry:
  br label %loop

loop:
  %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop ]
  %l = load i8, ptr %ptr.iv, align 1
  %a = add i8 %l, 1
  store i8 %a, ptr %ptr.iv, align 1
  %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 1
  %ec = icmp eq ptr %ptr.iv.next, %end
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

define void @ptr_induction(ptr noalias %dst, ptr %end) {
; VF2-LABEL: 'ptr_induction'
; VF2:  Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP5:%[0-9]+]]> * ir<8>
; VF2:  Cost of 0 for VF 2: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<8>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8>
; VF2:  Cost for VF 2: 5 (Estimated cost per lane: 2.5)
;
; VF4-LABEL: 'ptr_induction'
; VF4:  Cost of 1 for VF 4: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP5:%[0-9]+]]> * ir<8>
; VF4:  Cost of 0 for VF 4: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<8>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8>
; VF4:  Cost for VF 4: 5 (Estimated cost per lane: 1.25)
;
entry:
  br label %loop

loop:
  %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
  store i64 0, ptr %ptr.iv, align 8
  %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
  %ec = icmp eq ptr %ptr.iv.next, %end
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; Same as @ptr_induction, but the vector loop body executes at most once for
; VF 4, so there the canonical IV increment is simplified away and must not be
; costed.
define void @ptr_induction_tc_eq_vf(ptr noalias %dst) {
; VF2-LABEL: 'ptr_induction_tc_eq_vf'
; VF2:  Cost of 1 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP4:%[0-9]+]]> * ir<8>
; VF2:  Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<8>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 0 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8>
; VF2:  Cost for VF 2: 5 (Estimated cost per lane: 2.5)
;
; VF4-LABEL: 'ptr_induction_tc_eq_vf'
; VF4:  Cost of 1 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP4:%[0-9]+]]> * ir<8>
; VF4:  Cost of 0 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<8>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 0 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8>
; VF4:  Cost for VF 4: 3 (Estimated cost per lane: 0.75)
;
entry:
  br label %loop

loop:
  %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  store i64 0, ptr %ptr.iv, align 8
  %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 4
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; Test case where %ptr.iv remains a wide pointer induction, while %arr.iv is scalarized.
define void @ptr_induction_widened(ptr noalias %dst, ptr noalias %arr, ptr %end) {
; VF2-LABEL: 'ptr_induction_widened'
; VF2:  Cost of 1 for VF 2: EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%dst>, ir<8>, vp<[[VP1:%[0-9]+]]>
; VF2:  Cost of 1 for VF 2: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6:%[0-9]+]]> * ir<8>
; VF2:  Cost of 0 for VF 2: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<8>, vp<[[VP0:%[0-9]+]]>
; VF2:  Cost of 1 for VF 2: canonical IV increment
; VF2:  Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8>
; VF2:  Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<%arr> + vp<[[VP2]]> * ir<8>
; VF2:  Cost for VF 2: 7 (Estimated cost per lane: 3.5)
;
; VF4-LABEL: 'ptr_induction_widened'
; VF4:  Cost of 1 for VF 4: EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%dst>, ir<8>, vp<[[VP1:%[0-9]+]]>
; VF4:  Cost of 1 for VF 4: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6:%[0-9]+]]> * ir<8>
; VF4:  Cost of 0 for VF 4: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<8>, vp<[[VP0:%[0-9]+]]>
; VF4:  Cost of 1 for VF 4: canonical IV increment
; VF4:  Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8>
; VF4:  Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<%arr> + vp<[[VP2]]> * ir<8>
; VF4:  Cost for VF 4: 7 (Estimated cost per lane: 1.75)
;
entry:
  br label %loop

loop:
  %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
  %arr.iv = phi ptr [ %arr, %entry ], [ %arr.iv.next, %loop ]
  store ptr %ptr.iv, ptr %arr.iv, align 8
  store i64 0, ptr %ptr.iv, align 8
  %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1
  %arr.iv.next = getelementptr inbounds ptr, ptr %arr.iv, i64 1
  %ec = icmp eq ptr %ptr.iv.next, %end
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}