| ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "(Cost of .*(WIDEN-INDUCTION|SCALAR-STEPS|DERIVED-IV|WIDEN-POINTER-INDUCTION))|(canonical IV increment)|(Cost for VF)" --filter-out-after "LV: (Selecting|Using user) VF" --version 6 |
| ; REQUIRES: asserts |
| ; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck --check-prefix=VF2 %s |
| ; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck --check-prefix=VF4 %s |
| |
| define void @int_induction(ptr noalias %dst, i64 %n) { |
| ; VF2-LABEL: 'int_induction' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost for VF 2: 6 (Estimated cost per lane: 3) |
| ; |
| ; VF4-LABEL: 'int_induction' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost for VF 4: 6 (Estimated cost per lane: 1.5) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %gep = getelementptr inbounds i64, ptr %dst, i64 %iv |
| store i64 %iv, ptr %gep, align 8 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| define void @fp_induction(ptr noalias %dst, i64 %n) { |
| ; VF2-LABEL: 'fp_induction' |
| ; VF2: Cost of 2 for VF 2: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 1 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF2: Cost for VF 2: 6 (Estimated cost per lane: 3) |
| ; |
| ; VF4-LABEL: 'fp_induction' |
| ; VF4: Cost of 2 for VF 4: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 1 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF4: Cost for VF 4: 6 (Estimated cost per lane: 1.5) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ] |
| %gep = getelementptr inbounds float, ptr %dst, i64 %iv |
| store float %fp.iv, ptr %gep, align 4 |
| %fp.iv.next = fadd fast float %fp.iv, 1.000000e+00 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| define void @trunc_induction(ptr noalias %dst, i64 %n) { |
| ; VF2-LABEL: 'trunc_induction' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> (truncated to i32) |
| ; VF2: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost for VF 2: 6 (Estimated cost per lane: 3) |
| ; |
| ; VF4-LABEL: 'trunc_induction' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> (truncated to i32) |
| ; VF4: Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost for VF 4: 6 (Estimated cost per lane: 1.5) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %iv.trunc = trunc i64 %iv to i32 |
| %gep = getelementptr inbounds i32, ptr %dst, i64 %iv |
| store i32 %iv.trunc, ptr %gep, align 4 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| define void @trunc_and_wide_induction(ptr noalias %dst, ptr noalias %dst2, i64 %n) { |
| ; VF2-LABEL: 'trunc_and_wide_induction' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 2 for VF 2: ir<%iv>.1 = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]> (truncated to i32) |
| ; VF2: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost for VF 2: 9 (Estimated cost per lane: 4.5) |
| ; |
| ; VF4-LABEL: 'trunc_and_wide_induction' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 2 for VF 4: ir<%iv>.1 = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]> (truncated to i32) |
| ; VF4: Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost for VF 4: 9 (Estimated cost per lane: 2.25) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %iv.trunc = trunc i64 %iv to i32 |
| %gep = getelementptr inbounds i32, ptr %dst, i64 %iv |
| store i32 %iv.trunc, ptr %gep, align 4 |
| %gep2 = getelementptr inbounds i64, ptr %dst2, i64 %iv |
| store i64 %iv, ptr %gep2, align 8 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; All users of the FP induction need scalars, so no wide induction is generated |
| ; and the scalar steps materialize one FAdd per lane, except for the first. |
| define void @fp_induction_scalar_users(ptr noalias %dst, i64 %n) { |
| ; VF2-LABEL: 'fp_induction_scalar_users' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 1 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF2: Cost for VF 2: 10 (Estimated cost per lane: 5) |
| ; |
| ; VF4-LABEL: 'fp_induction_scalar_users' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 1 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 1 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF4: Cost for VF 4: 14 (Estimated cost per lane: 3.5) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ] |
| %idx = mul i64 %iv, 3 |
| %gep = getelementptr inbounds float, ptr %dst, i64 %idx |
| store float %fp.iv, ptr %gep, align 4 |
| %fp.iv.next = fadd fast float %fp.iv, 1.000000e+00 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; Same as @fp_induction_scalar_users, but for a decreasing induction, which uses |
| ; FSub to compute the per-lane values. |
| define void @fp_induction_fsub_scalar_users(ptr noalias %dst, i64 %n) { |
| ; VF2-LABEL: 'fp_induction_fsub_scalar_users' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 2 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<1.000000e+02> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 2 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<1.000000e+02> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF2: Cost for VF 2: 11 (Estimated cost per lane: 5.5) |
| ; |
| ; VF4-LABEL: 'fp_induction_fsub_scalar_users' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 2 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<1.000000e+02> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 2 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<1.000000e+02> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF4: Cost for VF 4: 15 (Estimated cost per lane: 3.75) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ 1.000000e+02, %entry ], [ %fp.iv.next, %loop ] |
| %idx = mul i64 %iv, 3 |
| %gep = getelementptr inbounds float, ptr %dst, i64 %idx |
| store float %fp.iv, ptr %gep, align 4 |
| %fp.iv.next = fsub fast float %fp.iv, 1.000000e+00 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; Same as @fp_induction_scalar_users, but with a loop-invariant step. |
| define void @fp_induction_invariant_step_scalar_users(ptr noalias %dst, float %step, i64 %n) { |
| ; VF2-LABEL: 'fp_induction_invariant_step_scalar_users' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 3 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<%step> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<%step>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 3 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<%step> |
| ; VF2: Cost for VF 2: 12 (Estimated cost per lane: 6) |
| ; |
| ; VF4-LABEL: 'fp_induction_invariant_step_scalar_users' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<%step> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<%step>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 3 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<%step> |
| ; VF4: Cost for VF 4: 16 (Estimated cost per lane: 4) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ] |
| %idx = mul i64 %iv, 3 |
| %gep = getelementptr inbounds float, ptr %dst, i64 %idx |
| store float %fp.iv, ptr %gep, align 4 |
| %fp.iv.next = fadd fast float %fp.iv, %step |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; Same as @fp_induction_scalar_users (fadd, unit step, zero start), but |
| ; without fast-math flags. The add is still expected to fold away: with a |
| ; unit step the DERIVED-IV's addend is exactly sitofp(<the non-negative |
| ; canonical/scalar IV>), which can never be -0.0, so "fadd 0.0, X" folds to X |
| ; regardless of fast-math flags. |
| define void @fp_induction_no_fastmath_unit_step_zero_start(ptr noalias %dst, i64 %n) { |
| ; VF2-LABEL: 'fp_induction_no_fastmath_unit_step_zero_start' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 1 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF2: Cost for VF 2: 10 (Estimated cost per lane: 5) |
| ; |
| ; VF4-LABEL: 'fp_induction_no_fastmath_unit_step_zero_start' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 1 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 1 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF4: Cost for VF 4: 14 (Estimated cost per lane: 3.5) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ] |
| %idx = mul i64 %iv, 3 |
| %gep = getelementptr inbounds float, ptr %dst, i64 %idx |
| store float %fp.iv, ptr %gep, align 4 |
| %fp.iv.next = fadd float %fp.iv, 1.000000e+00 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; Same as @fp_induction_no_fastmath_unit_step_zero_start, but with a |
| ; loop-invariant, sign-unknown step. Unlike the unit-step case, the add is |
| ; not known to fold away here: with a non-unit step the DERIVED-IV's addend |
| ; is Step * sitofp(Index), whose sign depends on the unknown sign of Step, so |
| ; it may be -0.0, and "fadd 0.0, X" doesn't fold to X in that case. |
| define void @fp_induction_no_fastmath_zero_start_invariant_step(ptr noalias %dst, float %step, i64 %n) { |
| ; VF2-LABEL: 'fp_induction_no_fastmath_zero_start_invariant_step' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 3 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<%step> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<%step>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 3 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<%step> |
| ; VF2: Cost for VF 2: 12 (Estimated cost per lane: 6) |
| ; |
| ; VF4-LABEL: 'fp_induction_no_fastmath_zero_start_invariant_step' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<%step> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<%step>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 3 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<%step> |
| ; VF4: Cost for VF 4: 16 (Estimated cost per lane: 4) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ] |
| %idx = mul i64 %iv, 3 |
| %gep = getelementptr inbounds float, ptr %dst, i64 %idx |
| store float %fp.iv, ptr %gep, align 4 |
| %fp.iv.next = fadd float %fp.iv, %step |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; FSub never folds away for a zero start, regardless of the step: "fsub 0.0, |
| ; X" isn't X, at best it simplifies to "fneg X", which is still a real |
| ; instruction. |
| define void @fp_induction_fsub_zero_start(ptr noalias %dst, i64 %n) { |
| ; VF2-LABEL: 'fp_induction_fsub_zero_start' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 2 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 2 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF2: Cost for VF 2: 11 (Estimated cost per lane: 5.5) |
| ; |
| ; VF4-LABEL: 'fp_induction_fsub_zero_start' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 2 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 2 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF4: Cost for VF 4: 15 (Estimated cost per lane: 3.75) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ] |
| %idx = mul i64 %iv, 3 |
| %gep = getelementptr inbounds float, ptr %dst, i64 %idx |
| store float %fp.iv, ptr %gep, align 4 |
| %fp.iv.next = fsub fast float %fp.iv, 1.000000e+00 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; Start of -0.0 so the FAdd can be folded. |
| define void @fp_induction_neg_zero_start_neg_step(ptr noalias %dst, i64 %n) { |
| ; VF2-LABEL: 'fp_induction_neg_zero_start_neg_step' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 2 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<-0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<-2.000000e+00> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<-2.000000e+00>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 2 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<-0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<-2.000000e+00> |
| ; VF2: Cost for VF 2: 11 (Estimated cost per lane: 5.5) |
| ; |
| ; VF4-LABEL: 'fp_induction_neg_zero_start_neg_step' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 2 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<-0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<-2.000000e+00> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<-2.000000e+00>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 2 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<-0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<-2.000000e+00> |
| ; VF4: Cost for VF 4: 15 (Estimated cost per lane: 3.75) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ -0.000000e+00, %entry ], [ %fp.iv.next, %loop ] |
| %idx = mul i64 %iv, 3 |
| %gep = getelementptr inbounds float, ptr %dst, i64 %idx |
| store float %fp.iv, ptr %gep, align 4 |
| %fp.iv.next = fadd float %fp.iv, -2.000000e+00 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; Same as @fp_induction_neg_zero_start_neg_step, but with a +0.0 start: the |
| ; FAdd cannot be folded. |
| define void @fp_induction_pos_zero_start_neg_step(ptr noalias %dst, i64 %n) { |
| ; VF2-LABEL: 'fp_induction_pos_zero_start_neg_step' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 3 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<-2.000000e+00> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<-2.000000e+00>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 3 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<-2.000000e+00> |
| ; VF2: Cost for VF 2: 12 (Estimated cost per lane: 6) |
| ; |
| ; VF4-LABEL: 'fp_induction_pos_zero_start_neg_step' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<-2.000000e+00> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<-2.000000e+00>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 3 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<-2.000000e+00> |
| ; VF4: Cost for VF 4: 16 (Estimated cost per lane: 4) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ] |
| %idx = mul i64 %iv, 3 |
| %gep = getelementptr inbounds float, ptr %dst, i64 %idx |
| store float %fp.iv, ptr %gep, align 4 |
| %fp.iv.next = fadd float %fp.iv, -2.000000e+00 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; Same as @fp_induction_pos_zero_start_neg_step, but with a positive step. |
| define void @fp_induction_pos_zero_start_pos_step(ptr noalias %dst, i64 %n) { |
| ; VF2-LABEL: 'fp_induction_pos_zero_start_pos_step' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 3 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<2.000000e+00> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<2.000000e+00>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 3 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<2.000000e+00> |
| ; VF2: Cost for VF 2: 12 (Estimated cost per lane: 6) |
| ; |
| ; VF4-LABEL: 'fp_induction_pos_zero_start_pos_step' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4:%[0-9]+]]> * ir<2.000000e+00> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<2.000000e+00>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 3 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<2.000000e+00> |
| ; VF4: Cost for VF 4: 16 (Estimated cost per lane: 4) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ] |
| %idx = mul i64 %iv, 3 |
| %gep = getelementptr inbounds float, ptr %dst, i64 %idx |
| store float %fp.iv, ptr %gep, align 4 |
| %fp.iv.next = fadd float %fp.iv, 2.000000e+00 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; A non-zero start value never folds, so the FAdd is charged even though the |
| ; unit step folds the FMul away. |
| define void @fp_induction_invariant_start(ptr noalias %dst, float %start, i64 %n) { |
| ; VF2-LABEL: 'fp_induction_invariant_start' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 2 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 2 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF2: Cost for VF 2: 11 (Estimated cost per lane: 5.5) |
| ; |
| ; VF4-LABEL: 'fp_induction_invariant_start' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 2 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP4:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 2 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF4: Cost for VF 4: 15 (Estimated cost per lane: 3.75) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ %start, %entry ], [ %fp.iv.next, %loop ] |
| %idx = mul i64 %iv, 3 |
| %gep = getelementptr inbounds float, ptr %dst, i64 %idx |
| store float %fp.iv, ptr %gep, align 4 |
| %fp.iv.next = fadd float %fp.iv, 1.000000e+00 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; The FP induction is only used outside the loop, so it is dead in the vector |
| ; loop and nothing is charged for it. |
| define float @fp_induction_used_outside_loop_only(ptr noalias %dst, i64 %n) { |
| ; VF2-LABEL: 'fp_induction_used_outside_loop_only' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost for VF 2: 8 (Estimated cost per lane: 4) |
| ; |
| ; VF4-LABEL: 'fp_induction_used_outside_loop_only' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost for VF 4: 8 (Estimated cost per lane: 2) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ] |
| %gep = getelementptr inbounds i64, ptr %dst, i64 %iv |
| store i64 %iv, ptr %gep, align 8 |
| %fp.iv.next = fadd fast float %fp.iv, 1.000000e+00 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret float %fp.iv.next |
| } |
| |
| ; The FP induction has both wide and scalar users, so both the wide induction |
| ; and the scalar steps are generated and need to be accounted for. |
| define void @fp_induction_wide_and_scalar_users(ptr noalias %dst, ptr noalias %dst2, i64 %n) { |
| ; VF2-LABEL: 'fp_induction_wide_and_scalar_users' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 2 for VF 2: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF2: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4]]> * ir<1.000000e+00> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 1 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF2: Cost for VF 2: 13 (Estimated cost per lane: 6.5) |
| ; |
| ; VF4-LABEL: 'fp_induction_wide_and_scalar_users' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 2 for VF 4: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF4: Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4]]> * ir<1.000000e+00> |
| ; VF4: Cost of 3 for VF 4: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 1 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF4: Cost for VF 4: 17 (Estimated cost per lane: 4.25) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ] |
| %gep = getelementptr inbounds float, ptr %dst, i64 %iv |
| store float %fp.iv, ptr %gep, align 4 |
| %idx = mul i64 %iv, 3 |
| %gep2 = getelementptr inbounds float, ptr %dst2, i64 %idx |
| store float %fp.iv, ptr %gep2, align 4 |
| %fp.iv.next = fadd fast float %fp.iv, 1.000000e+00 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; The scalar steps are inside a replicate region, for which no cost is computed |
| ; yet. |
| define void @fp_induction_predicated(ptr noalias %dst, ptr noalias %cond, i64 %n) { |
| ; VF2-LABEL: 'fp_induction_predicated' |
| ; VF2: Cost of 2 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4]]> * ir<1.000000e+00> |
| ; VF2: Cost of 0.5 for VF 2: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 1 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF2: Cost for VF 2: 12.5 (Estimated cost per lane: 6) |
| ; |
| ; VF4-LABEL: 'fp_induction_predicated' |
| ; VF4: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP4]]> * ir<1.000000e+00> |
| ; VF4: Cost of 1.5 for VF 4: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1.000000e+00>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 1 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF4: Cost for VF 4: 16.5 (Estimated cost per lane: 4) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] |
| %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %latch ] |
| %c.gep = getelementptr inbounds i8, ptr %cond, i64 %iv |
| %c = load i8, ptr %c.gep, align 1 |
| %c.0 = icmp eq i8 %c, 0 |
| br i1 %c.0, label %latch, label %then |
| |
| then: |
| %idx = mul i64 %iv, 3 |
| %gep = getelementptr inbounds float, ptr %dst, i64 %idx |
| store float %fp.iv, ptr %gep, align 4 |
| br label %latch |
| |
| latch: |
| %fp.iv.next = fadd fast float %fp.iv, 1.000000e+00 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; The vector loop is executed exactly once for VF 4, so the induction increments |
| ; and the canonical IV increment are all simplified away. |
| define void @fp_induction_tc_eq_4(ptr noalias %dst) { |
| ; VF2-LABEL: 'fp_induction_tc_eq_4' |
| ; VF2: Cost of 2 for VF 2: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 1 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF2: Cost for VF 2: 6 (Estimated cost per lane: 3) |
| ; |
| ; VF4-LABEL: 'fp_induction_tc_eq_4' |
| ; VF4: Cost of 0 for VF 4: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2:%[0-9]+]]> * ir<1.000000e+00> |
| ; VF4: Cost for VF 4: 2 (Estimated cost per lane: 0.5) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ] |
| %gep = getelementptr inbounds float, ptr %dst, i64 %iv |
| store float %fp.iv, ptr %gep, align 4 |
| %fp.iv.next = fadd fast float %fp.iv, 1.000000e+00 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, 4 |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; The loop is advanced by both an FP and a pointer induction. Neither induction |
| ; is costed by the legacy cost model any more; the scalarized pointer induction |
| ; is covered by its DERIVED-IV and SCALAR-STEPS recipes. |
| define void @fp_and_ptr_induction(ptr noalias %dst, ptr %end) { |
| ; VF2-LABEL: 'fp_and_ptr_induction' |
| ; VF2: Cost of 2 for VF 2: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6:%[0-9]+]]> * ir<4> |
| ; VF2: Cost of 0 for VF 2: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<4>, vp<[[VP0]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<4> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2]]> * ir<1.000000e+00> |
| ; VF2: Cost for VF 2: 7 (Estimated cost per lane: 3.5) |
| ; |
| ; VF4-LABEL: 'fp_and_ptr_induction' |
| ; VF4: Cost of 2 for VF 4: ir<%fp.iv> = WIDEN-INDUCTION fast ir<0.000000e+00>, ir<1.000000e+00>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 1 for VF 4: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6:%[0-9]+]]> * ir<4> |
| ; VF4: Cost of 0 for VF 4: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<4>, vp<[[VP0]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<4> |
| ; VF4: Cost of 1 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0.000000e+00> + vp<[[VP2]]> * ir<1.000000e+00> |
| ; VF4: Cost for VF 4: 7 (Estimated cost per lane: 1.75) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ] |
| %fp.iv = phi float [ 0.000000e+00, %entry ], [ %fp.iv.next, %loop ] |
| store float %fp.iv, ptr %ptr.iv, align 4 |
| %fp.iv.next = fadd fast float %fp.iv, 1.000000e+00 |
| %ptr.iv.next = getelementptr inbounds float, ptr %ptr.iv, i64 1 |
| %ec = icmp eq ptr %ptr.iv.next, %end |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; Same as @fp_and_ptr_induction, but the loop is only advanced by a pointer |
| ; induction. The canonical IV increment is still charged, even though the GEP |
| ; incrementing the pointer induction is free in the scalar cost. |
| define void @ptr_induction_only(ptr %start, ptr %end) { |
| ; VF2-LABEL: 'ptr_induction_only' |
| ; VF2: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5:%[0-9]+]]>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP2:%[0-9]+]]> * ir<1> |
| ; VF2: Cost for VF 2: 6 (Estimated cost per lane: 3) |
| ; |
| ; VF4-LABEL: 'ptr_induction_only' |
| ; VF4: Cost of 0 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5:%[0-9]+]]>, ir<1>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%start> + vp<[[VP2:%[0-9]+]]> * ir<1> |
| ; VF4: Cost for VF 4: 6 (Estimated cost per lane: 1.5) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop ] |
| %l = load i8, ptr %ptr.iv, align 1 |
| %a = add i8 %l, 1 |
| store i8 %a, ptr %ptr.iv, align 1 |
| %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 1 |
| %ec = icmp eq ptr %ptr.iv.next, %end |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| define void @ptr_induction(ptr noalias %dst, ptr %end) { |
| ; VF2-LABEL: 'ptr_induction' |
| ; VF2: Cost of 1 for VF 2: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP5:%[0-9]+]]> * ir<8> |
| ; VF2: Cost of 0 for VF 2: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<8>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8> |
| ; VF2: Cost for VF 2: 5 (Estimated cost per lane: 2.5) |
| ; |
| ; VF4-LABEL: 'ptr_induction' |
| ; VF4: Cost of 1 for VF 4: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP5:%[0-9]+]]> * ir<8> |
| ; VF4: Cost of 0 for VF 4: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<8>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8> |
| ; VF4: Cost for VF 4: 5 (Estimated cost per lane: 1.25) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ] |
| store i64 0, ptr %ptr.iv, align 8 |
| %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1 |
| %ec = icmp eq ptr %ptr.iv.next, %end |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; Same as @ptr_induction, but the vector loop body executes at most once for |
| ; VF 4, so there the canonical IV increment is simplified away and must not be |
| ; costed. |
| define void @ptr_induction_tc_eq_vf(ptr noalias %dst) { |
| ; VF2-LABEL: 'ptr_induction_tc_eq_vf' |
| ; VF2: Cost of 1 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP4:%[0-9]+]]> * ir<8> |
| ; VF2: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<8>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 0 for VF 2: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8> |
| ; VF2: Cost for VF 2: 5 (Estimated cost per lane: 2.5) |
| ; |
| ; VF4-LABEL: 'ptr_induction_tc_eq_vf' |
| ; VF4: Cost of 1 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP4:%[0-9]+]]> * ir<8> |
| ; VF4: Cost of 0 for VF 4: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<8>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 0 for VF 4: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8> |
| ; VF4: Cost for VF 4: 3 (Estimated cost per lane: 0.75) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ] |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| store i64 0, ptr %ptr.iv, align 8 |
| %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, 4 |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; Test case where %ptr.iv remains a wide pointer induction, while %arr.iv is scalarized. |
| define void @ptr_induction_widened(ptr noalias %dst, ptr noalias %arr, ptr %end) { |
| ; VF2-LABEL: 'ptr_induction_widened' |
| ; VF2: Cost of 1 for VF 2: EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%dst>, ir<8>, vp<[[VP1:%[0-9]+]]> |
| ; VF2: Cost of 1 for VF 2: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6:%[0-9]+]]> * ir<8> |
| ; VF2: Cost of 0 for VF 2: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<8>, vp<[[VP0:%[0-9]+]]> |
| ; VF2: Cost of 1 for VF 2: canonical IV increment |
| ; VF2: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8> |
| ; VF2: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<%arr> + vp<[[VP2]]> * ir<8> |
| ; VF2: Cost for VF 2: 7 (Estimated cost per lane: 3.5) |
| ; |
| ; VF4-LABEL: 'ptr_induction_widened' |
| ; VF4: Cost of 1 for VF 4: EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%dst>, ir<8>, vp<[[VP1:%[0-9]+]]> |
| ; VF4: Cost of 1 for VF 4: vp<[[VP7:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP6:%[0-9]+]]> * ir<8> |
| ; VF4: Cost of 0 for VF 4: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<8>, vp<[[VP0:%[0-9]+]]> |
| ; VF4: Cost of 1 for VF 4: canonical IV increment |
| ; VF4: Cost of 0 for VF 4: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%dst> + vp<[[VP2:%[0-9]+]]> * ir<8> |
| ; VF4: Cost of 0 for VF 4: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<%arr> + vp<[[VP2]]> * ir<8> |
| ; VF4: Cost for VF 4: 7 (Estimated cost per lane: 1.75) |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ] |
| %arr.iv = phi ptr [ %arr, %entry ], [ %arr.iv.next, %loop ] |
| store ptr %ptr.iv, ptr %arr.iv, align 8 |
| store i64 0, ptr %ptr.iv, align 8 |
| %ptr.iv.next = getelementptr inbounds i64, ptr %ptr.iv, i64 1 |
| %arr.iv.next = getelementptr inbounds ptr, ptr %arr.iv, i64 1 |
| %ec = icmp eq ptr %ptr.iv.next, %end |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
