Google GitGoogle Git
Sign in
llvm/llvm-project/330b49f03ff4bf8005c9dc95e6ffc37b341bbbd2/./llvm/test/Transforms/LoopVectorize/AArch64/cmp_cost.ll
blob: a25edccb60514507f1ec3fc34b4d390cc4e66b27 [file]
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of" --filter "Selecting VF" --filter-out-after "Selecting VF" --version 6
; REQUIRES: asserts
; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s
target triple = "aarch64-none-elf"
; fmaxnum reduction involves generating a vector unordered fcmp which is
; comparatively expensive. This makes it only profitable for VF 4, so we can
; vectorize when the data type is f32, but not when it's f64 where the max VF
; is 2.
define float @fmaxnum_reduction_f32(float %base, i32 %n) {
; CHECK-LABEL: 'fmaxnum_reduction_f32'
; CHECK: Cost of 1 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: WIDEN-REDUCTION-PHI ir<%max> = phi (fmaxnum) ir<-1.000000e+07>, ir<%max.next>
; CHECK: Cost of 1 for VF 2: WIDEN-CAST ir<%iv.f> = sitofp ir<%iv> to float
; CHECK: Cost of 1 for VF 2: WIDEN ir<%v> = fadd ir<%base>, ir<%iv.f>
; CHECK: Cost of 1 for VF 2: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, ir<%v>)
; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3:%[0-9]+]]>, vp<[[VP1:%[0-9]+]]>
; CHECK: Cost of 3 for VF 2: EMIT vp<[[VP4:%[0-9]+]]> = fcmp uno ir<%v>, ir<%v>
; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP5:%[0-9]+]]> = any-of vp<[[VP4]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP6:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP7]]>
; CHECK: Cost of 0 for VF 2: vector loop backedge
; CHECK: Cost of 1 for VF 2: canonical IV increment
; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10:%[0-9]+]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP11:%[0-9]+]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 2: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %iv.f = sitofp i32 %iv to float
; CHECK: Cost of 0 for VF 2: IR %v = fadd float %base, %iv.f
; CHECK: Cost of 0 for VF 2: IR %max.next = call float @llvm.maxnum.f32(float %max, float %v)
; CHECK: Cost of 0 for VF 2: IR %iv.next = add nuw nsw i32 %iv, 1
; CHECK: Cost of 0 for VF 2: IR %ec = icmp eq i32 %iv.next, %n
; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP9:%[0-9]+]]> = select vp<[[VP5]]>, ir<%max>, ir<%max.next>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP5]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP12]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP13]]>
; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
; CHECK: Cost of 1 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 4: WIDEN-REDUCTION-PHI ir<%max> = phi (fmaxnum) ir<-1.000000e+07>, ir<%max.next>
; CHECK: Cost of 1 for VF 4: WIDEN-CAST ir<%iv.f> = sitofp ir<%iv> to float
; CHECK: Cost of 1 for VF 4: WIDEN ir<%v> = fadd ir<%base>, ir<%iv.f>
; CHECK: Cost of 1 for VF 4: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, ir<%v>)
; CHECK: Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK: Cost of 3 for VF 4: EMIT vp<[[VP4]]> = fcmp uno ir<%v>, ir<%v>
; CHECK: Cost of 2 for VF 4: EMIT vp<[[VP5]]> = any-of vp<[[VP4]]>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP6]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP7]]> = or vp<[[VP5]]>, vp<[[VP6]]>
; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<[[VP7]]>
; CHECK: Cost of 0 for VF 4: vector loop backedge
; CHECK: Cost of 1 for VF 4: canonical IV increment
; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP11]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 4: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 4: IR %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
; CHECK: Cost of 0 for VF 4: IR %iv.f = sitofp i32 %iv to float
; CHECK: Cost of 0 for VF 4: IR %v = fadd float %base, %iv.f
; CHECK: Cost of 0 for VF 4: IR %max.next = call float @llvm.maxnum.f32(float %max, float %v)
; CHECK: Cost of 0 for VF 4: IR %iv.next = add nuw nsw i32 %iv, 1
; CHECK: Cost of 0 for VF 4: IR %ec = icmp eq i32 %iv.next, %n
; CHECK: Cost of 2 for VF 4: EMIT vp<[[VP9]]> = select vp<[[VP5]]>, ir<%max>, ir<%max.next>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
; CHECK: Cost of 1 for VF 4: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP12]]> = not vp<[[VP5]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP13]]> = and vp<%cmp.n>, vp<[[VP12]]>
; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<[[VP13]]>
; CHECK: Cost of 0 for VF 4: IR %max.next.lcssa = phi float [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
; CHECK: LV: Selecting VF: 4.
;
entry:
br label %loop
loop:
%iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
%max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ]
%iv.f = sitofp i32 %iv to float
%v = fadd float %base, %iv.f
%max.next = call float @llvm.maxnum.f32(float %max, float %v)
%iv.next = add nuw nsw i32 %iv, 1
%ec = icmp eq i32 %iv.next, %n
br i1 %ec, label %exit, label %loop
exit:
ret float %max.next
}
define double @fmaxnum_reduction_f64(double %base, i64 %n) {
; CHECK-LABEL: 'fmaxnum_reduction_f64'
; CHECK: Cost of 1 for VF 2: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: WIDEN-REDUCTION-PHI ir<%max> = phi (fmaxnum) ir<-1.000000e+07>, ir<%max.next>
; CHECK: Cost of 1 for VF 2: WIDEN-CAST ir<%iv.f> = sitofp ir<%iv> to double
; CHECK: Cost of 1 for VF 2: WIDEN ir<%v> = fadd ir<%base>, ir<%iv.f>
; CHECK: Cost of 1 for VF 2: WIDEN-INTRINSIC ir<%max.next> = call llvm.maxnum(ir<%max>, ir<%v>)
; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3:%[0-9]+]]>, vp<[[VP1:%[0-9]+]]>
; CHECK: Cost of 3 for VF 2: EMIT vp<[[VP4:%[0-9]+]]> = fcmp uno ir<%v>, ir<%v>
; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP5:%[0-9]+]]> = any-of vp<[[VP4]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP6:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP7]]>
; CHECK: Cost of 0 for VF 2: vector loop backedge
; CHECK: Cost of 1 for VF 2: canonical IV increment
; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10:%[0-9]+]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP11:%[0-9]+]]>, middle.block ], [ ir<-1.000000e+07>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 2: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %max = phi double [ -1.000000e+07, %entry ], [ %max.next, %loop ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %iv.f = sitofp i64 %iv to double
; CHECK: Cost of 0 for VF 2: IR %v = fadd double %base, %iv.f
; CHECK: Cost of 0 for VF 2: IR %max.next = call double @llvm.maxnum.f64(double %max, double %v)
; CHECK: Cost of 0 for VF 2: IR %iv.next = add nuw nsw i64 %iv, 1
; CHECK: Cost of 0 for VF 2: IR %ec = icmp eq i64 %iv.next, %n
; CHECK: Cost of 2 for VF 2: EMIT vp<[[VP9:%[0-9]+]]> = select vp<[[VP5]]>, ir<%max>, ir<%max.next>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP10]]> = select vp<[[VP5]]>, vp<[[VP3]]>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP11]]> = compute-reduction-result (fmaxnum) vp<[[VP9]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = not vp<[[VP5]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = and vp<%cmp.n>, vp<[[VP12]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<[[VP13]]>
; CHECK: Cost of 0 for VF 2: IR %max.next.lcssa = phi double [ %max.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
; CHECK: LV: Selecting VF: 1.
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%max = phi double [ -1.000000e+07, %entry ], [ %max.next, %loop ]
%iv.f = sitofp i64 %iv to double
%v = fadd double %base, %iv.f
%max.next = call double @llvm.maxnum.f64(double %max, double %v)
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop
exit:
ret double %max.next
}
; Vectorization of switch involves creating an icmp for each switch case.
define i32 @switch_to_cmp(ptr %s, ptr %dst, i64 %n) {
; CHECK-LABEL: 'switch_to_cmp'
; CHECK: Cost of 0 for VF 2: forced scalar %gep = getelementptr i8, ptr %s, i64 %iv
; CHECK: Cost of 0 for VF 2: forced scalar %dst.gep = getelementptr i8, ptr %dst, i64 %iv
; CHECK: Cost of 0 for VF 2: WIDEN-REDUCTION-PHI ir<%c> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%c.next>
; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4:%[0-9]+]]>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: CLONE ir<%gep> = getelementptr ir<%s>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer i8, ir<%gep>, ir<1>
; CHECK: Cost of 4 for VF 2: WIDEN ir<%l> = load vp<[[VP6]]>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP7:%[0-9]+]]> = icmp eq ir<%l>, ir<1>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP8:%[0-9]+]]> = icmp eq ir<%l>, ir<2>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP9:%[0-9]+]]> = icmp eq ir<%l>, ir<3>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq ir<%l>, ir<4>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP11:%[0-9]+]]> = icmp eq ir<%l>, ir<5>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP12:%[0-9]+]]> = icmp eq ir<%l>, ir<6>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP13:%[0-9]+]]> = icmp eq ir<%l>, ir<7>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq ir<%l>, ir<8>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP15:%[0-9]+]]> = icmp eq ir<%l>, ir<9>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP16:%[0-9]+]]> = icmp eq ir<%l>, ir<10>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP17:%[0-9]+]]> = icmp eq ir<%l>, ir<11>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP18:%[0-9]+]]> = icmp eq ir<%l>, ir<12>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP19:%[0-9]+]]> = icmp eq ir<%l>, ir<13>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP20:%[0-9]+]]> = icmp eq ir<%l>, ir<14>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP21:%[0-9]+]]> = icmp eq ir<%l>, ir<15>
; CHECK: Cost of 1 for VF 2: EMIT vp<[[VP22:%[0-9]+]]> = icmp eq ir<%l>, ir<0>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP23:%[0-9]+]]> = or vp<[[VP7]]>, vp<[[VP8]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP24:%[0-9]+]]> = or vp<[[VP23]]>, vp<[[VP9]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP25:%[0-9]+]]> = or vp<[[VP24]]>, vp<[[VP10]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP26:%[0-9]+]]> = or vp<[[VP25]]>, vp<[[VP11]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP27:%[0-9]+]]> = or vp<[[VP26]]>, vp<[[VP12]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP28:%[0-9]+]]> = or vp<[[VP27]]>, vp<[[VP13]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP29:%[0-9]+]]> = or vp<[[VP28]]>, vp<[[VP14]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP30:%[0-9]+]]> = or vp<[[VP29]]>, vp<[[VP15]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP31:%[0-9]+]]> = or vp<[[VP30]]>, vp<[[VP16]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP32:%[0-9]+]]> = or vp<[[VP31]]>, vp<[[VP17]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP33:%[0-9]+]]> = or vp<[[VP32]]>, vp<[[VP18]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP34:%[0-9]+]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP35:%[0-9]+]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP36:%[0-9]+]]> = or vp<[[VP35]]>, vp<[[VP21]]>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%c.4> = add ir<%c>, ir<4>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%c.1> = add ir<%c>, ir<1>
; CHECK: Cost of 7 for VF 2: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 2: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP37:%[0-9]+]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
; CHECK: Cost of 4 for VF 2: WIDEN store vp<[[VP37]]>, ir<%l>
; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1:%[0-9]+]]>
; CHECK: Cost of 1 for VF 2: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: vector loop backedge
; CHECK: Cost of 1 for VF 2: canonical IV increment
; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP39:%[0-9]+]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 2: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %c = phi i32 [ 0, %entry ], [ %c.next, %loop.latch ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %gep = getelementptr i8, ptr %s, i64 %iv
; CHECK: Cost of 0 for VF 2: IR %l = load i8, ptr %gep, align 1
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP3]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK: Cost of 0 for VF 2: EMIT vp<[[VP39]]> = compute-reduction-result (add) ir<%c.next>
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<%cmp.n>
; CHECK: Cost of 0 for VF 2: IR %c.next.lcssa = phi i32 [ %c.next, %loop.latch ] (extra operand: vp<[[VP39]]> from middle.block)
; CHECK: Cost of 0 for VF 4: forced scalar %gep = getelementptr i8, ptr %s, i64 %iv
; CHECK: Cost of 0 for VF 4: forced scalar %dst.gep = getelementptr i8, ptr %dst, i64 %iv
; CHECK: Cost of 0 for VF 4: WIDEN-REDUCTION-PHI ir<%c> = phi (add) vp<[[VP3]]>, ir<%c.next>
; CHECK: Cost of 0 for VF 4: vp<[[VP5]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 4: CLONE ir<%gep> = getelementptr ir<%s>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer i8, ir<%gep>, ir<1>
; CHECK: Cost of 2 for VF 4: WIDEN ir<%l> = load vp<[[VP6]]>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP7]]> = icmp eq ir<%l>, ir<1>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP8]]> = icmp eq ir<%l>, ir<2>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP9]]> = icmp eq ir<%l>, ir<3>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP10]]> = icmp eq ir<%l>, ir<4>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP11]]> = icmp eq ir<%l>, ir<5>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP12]]> = icmp eq ir<%l>, ir<6>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP13]]> = icmp eq ir<%l>, ir<7>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP14]]> = icmp eq ir<%l>, ir<8>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP15]]> = icmp eq ir<%l>, ir<9>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP16]]> = icmp eq ir<%l>, ir<10>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP17]]> = icmp eq ir<%l>, ir<11>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP18]]> = icmp eq ir<%l>, ir<12>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP19]]> = icmp eq ir<%l>, ir<13>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP20]]> = icmp eq ir<%l>, ir<14>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP21]]> = icmp eq ir<%l>, ir<15>
; CHECK: Cost of 1 for VF 4: EMIT vp<[[VP22]]> = icmp eq ir<%l>, ir<0>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP23]]> = or vp<[[VP7]]>, vp<[[VP8]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP24]]> = or vp<[[VP23]]>, vp<[[VP9]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP25]]> = or vp<[[VP24]]>, vp<[[VP10]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP26]]> = or vp<[[VP25]]>, vp<[[VP11]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP27]]> = or vp<[[VP26]]>, vp<[[VP12]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP28]]> = or vp<[[VP27]]>, vp<[[VP13]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP29]]> = or vp<[[VP28]]>, vp<[[VP14]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP30]]> = or vp<[[VP29]]>, vp<[[VP15]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP31]]> = or vp<[[VP30]]>, vp<[[VP16]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP32]]> = or vp<[[VP31]]>, vp<[[VP17]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP33]]> = or vp<[[VP32]]>, vp<[[VP18]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP34]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP35]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP36]]> = or vp<[[VP35]]>, vp<[[VP21]]>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%c.4> = add ir<%c>, ir<4>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%c.1> = add ir<%c>, ir<1>
; CHECK: Cost of 13 for VF 4: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 4: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP37]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
; CHECK: Cost of 2 for VF 4: WIDEN store vp<[[VP37]]>, ir<%l>
; CHECK: Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK: Cost of 1 for VF 4: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 4: vector loop backedge
; CHECK: Cost of 1 for VF 4: canonical IV increment
; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP39]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 4: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 4: IR %c = phi i32 [ 0, %entry ], [ %c.next, %loop.latch ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
; CHECK: Cost of 0 for VF 4: IR %gep = getelementptr i8, ptr %s, i64 %iv
; CHECK: Cost of 0 for VF 4: IR %l = load i8, ptr %gep, align 1
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP3]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK: Cost of 0 for VF 4: EMIT vp<[[VP39]]> = compute-reduction-result (add) ir<%c.next>
; CHECK: Cost of 1 for VF 4: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<%cmp.n>
; CHECK: Cost of 0 for VF 4: IR %c.next.lcssa = phi i32 [ %c.next, %loop.latch ] (extra operand: vp<[[VP39]]> from middle.block)
; CHECK: Cost of 0 for VF 8: forced scalar %gep = getelementptr i8, ptr %s, i64 %iv
; CHECK: Cost of 0 for VF 8: forced scalar %dst.gep = getelementptr i8, ptr %dst, i64 %iv
; CHECK: Cost of 0 for VF 8: WIDEN-REDUCTION-PHI ir<%c> = phi (add) vp<[[VP3]]>, ir<%c.next>
; CHECK: Cost of 0 for VF 8: vp<[[VP5]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 8: CLONE ir<%gep> = getelementptr ir<%s>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 8: vp<[[VP6]]> = vector-pointer i8, ir<%gep>, ir<1>
; CHECK: Cost of 1 for VF 8: WIDEN ir<%l> = load vp<[[VP6]]>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP7]]> = icmp eq ir<%l>, ir<1>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP8]]> = icmp eq ir<%l>, ir<2>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP9]]> = icmp eq ir<%l>, ir<3>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP10]]> = icmp eq ir<%l>, ir<4>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP11]]> = icmp eq ir<%l>, ir<5>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP12]]> = icmp eq ir<%l>, ir<6>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP13]]> = icmp eq ir<%l>, ir<7>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP14]]> = icmp eq ir<%l>, ir<8>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP15]]> = icmp eq ir<%l>, ir<9>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP16]]> = icmp eq ir<%l>, ir<10>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP17]]> = icmp eq ir<%l>, ir<11>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP18]]> = icmp eq ir<%l>, ir<12>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP19]]> = icmp eq ir<%l>, ir<13>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP20]]> = icmp eq ir<%l>, ir<14>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP21]]> = icmp eq ir<%l>, ir<15>
; CHECK: Cost of 1 for VF 8: EMIT vp<[[VP22]]> = icmp eq ir<%l>, ir<0>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP23]]> = or vp<[[VP7]]>, vp<[[VP8]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP24]]> = or vp<[[VP23]]>, vp<[[VP9]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP25]]> = or vp<[[VP24]]>, vp<[[VP10]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP26]]> = or vp<[[VP25]]>, vp<[[VP11]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP27]]> = or vp<[[VP26]]>, vp<[[VP12]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP28]]> = or vp<[[VP27]]>, vp<[[VP13]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP29]]> = or vp<[[VP28]]>, vp<[[VP14]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP30]]> = or vp<[[VP29]]>, vp<[[VP15]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP31]]> = or vp<[[VP30]]>, vp<[[VP16]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP32]]> = or vp<[[VP31]]>, vp<[[VP17]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP33]]> = or vp<[[VP32]]>, vp<[[VP18]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP34]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP35]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP36]]> = or vp<[[VP35]]>, vp<[[VP21]]>
; CHECK: Cost of 2 for VF 8: WIDEN ir<%c.4> = add ir<%c>, ir<4>
; CHECK: Cost of 2 for VF 8: WIDEN ir<%c.1> = add ir<%c>, ir<1>
; CHECK: Cost of 10 for VF 8: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 8: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 8: vp<[[VP37]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
; CHECK: Cost of 1 for VF 8: WIDEN store vp<[[VP37]]>, ir<%l>
; CHECK: Cost of 0 for VF 8: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK: Cost of 1 for VF 8: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 8: vector loop backedge
; CHECK: Cost of 1 for VF 8: canonical IV increment
; CHECK: Cost of 0 for VF 8: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 8: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP39]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 8: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 8: IR %c = phi i32 [ 0, %entry ], [ %c.next, %loop.latch ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
; CHECK: Cost of 0 for VF 8: IR %gep = getelementptr i8, ptr %s, i64 %iv
; CHECK: Cost of 0 for VF 8: IR %l = load i8, ptr %gep, align 1
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP3]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK: Cost of 0 for VF 8: EMIT vp<[[VP39]]> = compute-reduction-result (add) ir<%c.next>
; CHECK: Cost of 1 for VF 8: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 8: EMIT branch-on-cond vp<%cmp.n>
; CHECK: Cost of 0 for VF 8: IR %c.next.lcssa = phi i32 [ %c.next, %loop.latch ] (extra operand: vp<[[VP39]]> from middle.block)
; CHECK: Cost of 0 for VF 16: forced scalar %gep = getelementptr i8, ptr %s, i64 %iv
; CHECK: Cost of 0 for VF 16: forced scalar %dst.gep = getelementptr i8, ptr %dst, i64 %iv
; CHECK: Cost of 0 for VF 16: WIDEN-REDUCTION-PHI ir<%c> = phi (add) vp<[[VP3]]>, ir<%c.next>
; CHECK: Cost of 0 for VF 16: vp<[[VP5]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 16: CLONE ir<%gep> = getelementptr ir<%s>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 16: vp<[[VP6]]> = vector-pointer i8, ir<%gep>, ir<1>
; CHECK: Cost of 1 for VF 16: WIDEN ir<%l> = load vp<[[VP6]]>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP7]]> = icmp eq ir<%l>, ir<1>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP8]]> = icmp eq ir<%l>, ir<2>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP9]]> = icmp eq ir<%l>, ir<3>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP10]]> = icmp eq ir<%l>, ir<4>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP11]]> = icmp eq ir<%l>, ir<5>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP12]]> = icmp eq ir<%l>, ir<6>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP13]]> = icmp eq ir<%l>, ir<7>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP14]]> = icmp eq ir<%l>, ir<8>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP15]]> = icmp eq ir<%l>, ir<9>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP16]]> = icmp eq ir<%l>, ir<10>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP17]]> = icmp eq ir<%l>, ir<11>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP18]]> = icmp eq ir<%l>, ir<12>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP19]]> = icmp eq ir<%l>, ir<13>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP20]]> = icmp eq ir<%l>, ir<14>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP21]]> = icmp eq ir<%l>, ir<15>
; CHECK: Cost of 1 for VF 16: EMIT vp<[[VP22]]> = icmp eq ir<%l>, ir<0>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP23]]> = or vp<[[VP7]]>, vp<[[VP8]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP24]]> = or vp<[[VP23]]>, vp<[[VP9]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP25]]> = or vp<[[VP24]]>, vp<[[VP10]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP26]]> = or vp<[[VP25]]>, vp<[[VP11]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP27]]> = or vp<[[VP26]]>, vp<[[VP12]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP28]]> = or vp<[[VP27]]>, vp<[[VP13]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP29]]> = or vp<[[VP28]]>, vp<[[VP14]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP30]]> = or vp<[[VP29]]>, vp<[[VP15]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP31]]> = or vp<[[VP30]]>, vp<[[VP16]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP32]]> = or vp<[[VP31]]>, vp<[[VP17]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP33]]> = or vp<[[VP32]]>, vp<[[VP18]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP34]]> = or vp<[[VP33]]>, vp<[[VP19]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP35]]> = or vp<[[VP34]]>, vp<[[VP20]]>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP36]]> = or vp<[[VP35]]>, vp<[[VP21]]>
; CHECK: Cost of 4 for VF 16: WIDEN ir<%c.4> = add ir<%c>, ir<4>
; CHECK: Cost of 4 for VF 16: WIDEN ir<%c.1> = add ir<%c>, ir<1>
; CHECK: Cost of 20 for VF 16: BLEND ir<%c.next> = ir<%c> ir<%c.4>/vp<[[VP22]]> ir<%c.1>/vp<[[VP36]]>
; CHECK: Cost of 0 for VF 16: CLONE ir<%dst.gep> = getelementptr ir<%dst>, vp<[[VP5]]>
; CHECK: Cost of 0 for VF 16: vp<[[VP37]]> = vector-pointer i8, ir<%dst.gep>, ir<1>
; CHECK: Cost of 1 for VF 16: WIDEN store vp<[[VP37]]>, ir<%l>
; CHECK: Cost of 0 for VF 16: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK: Cost of 1 for VF 16: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 16: vector loop backedge
; CHECK: Cost of 1 for VF 16: canonical IV increment
; CHECK: Cost of 0 for VF 16: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 16: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP39]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 16: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 16: IR %c = phi i32 [ 0, %entry ], [ %c.next, %loop.latch ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)
; CHECK: Cost of 0 for VF 16: IR %gep = getelementptr i8, ptr %s, i64 %iv
; CHECK: Cost of 0 for VF 16: IR %l = load i8, ptr %gep, align 1
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP3]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK: Cost of 0 for VF 16: EMIT vp<[[VP39]]> = compute-reduction-result (add) ir<%c.next>
; CHECK: Cost of 1 for VF 16: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 16: EMIT branch-on-cond vp<%cmp.n>
; CHECK: Cost of 0 for VF 16: IR %c.next.lcssa = phi i32 [ %c.next, %loop.latch ] (extra operand: vp<[[VP39]]> from middle.block)
; CHECK: LV: Selecting VF: 16.
;
entry:
br label %loop.header
loop.header:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
%c = phi i32 [ 0, %entry ], [ %c.next, %loop.latch ]
%gep = getelementptr i8, ptr %s, i64 %iv
%l = load i8, ptr %gep
switch i8 %l, label %loop.latch [
i8 1, label %add.1
i8 2, label %add.1
i8 3, label %add.1
i8 4, label %add.1
i8 5, label %add.1
i8 6, label %add.1
i8 7, label %add.1
i8 8, label %add.1
i8 9, label %add.1
i8 10, label %add.1
i8 11, label %add.1
i8 12, label %add.1
i8 13, label %add.1
i8 14, label %add.1
i8 15, label %add.1
i8 0, label %add.4
]
add.1:
%c.1 = add nsw i32 %c, 1
br label %loop.latch
add.4:
%c.4 = add nsw i32 %c, 4
br label %loop.latch
loop.latch:
%c.next = phi i32 [ %c, %loop.header ], [ %c.1, %add.1 ], [ %c.4, %add.4 ]
%dst.gep = getelementptr i8, ptr %dst, i64 %iv
store i8 %l, ptr %dst.gep
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop.header
exit:
ret i32 %c.next
}
; The predicate of a blend's fcmp mask is passed to the cost model. AArch64 can
; lower a select fed by an fcmp olt using a FCMxx & BFI pair, while it cannot do
; so for fcmp uno, which is more expensive.
define void @blend_fcmp_olt_f32(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK-LABEL: 'blend_fcmp_olt_f32'
; CHECK: Cost of 0 for VF 2: forced scalar %src.gep = getelementptr inbounds float, ptr %src, i64 %iv
; CHECK: Cost of 0 for VF 2: forced scalar %dst.gep = getelementptr inbounds float, ptr %dst, i64 %iv
; CHECK: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: CLONE ir<%src.gep> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%c> = fcmp olt ir<%l>, ir<0.000000e+00>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
; CHECK: Cost of 1 for VF 2: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 2: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP6]]>, ir<%p>
; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK: Cost of 1 for VF 2: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: vector loop backedge
; CHECK: Cost of 1 for VF 2: canonical IV increment
; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 2: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %src.gep = getelementptr inbounds float, ptr %src, i64 %iv
; CHECK: Cost of 0 for VF 2: IR %l = load float, ptr %src.gep, align 4
; CHECK: Cost of 0 for VF 2: IR %c = fcmp olt float %l, 0.000000e+00
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<%cmp.n>
; CHECK: Cost of 0 for VF 4: forced scalar %src.gep = getelementptr inbounds float, ptr %src, i64 %iv
; CHECK: Cost of 0 for VF 4: forced scalar %dst.gep = getelementptr inbounds float, ptr %dst, i64 %iv
; CHECK: Cost of 0 for VF 4: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 4: CLONE ir<%src.gep> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%c> = fcmp olt ir<%l>, ir<0.000000e+00>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
; CHECK: Cost of 1 for VF 4: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 4: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP6]]>, ir<%p>
; CHECK: Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK: Cost of 1 for VF 4: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 4: vector loop backedge
; CHECK: Cost of 1 for VF 4: canonical IV increment
; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 4: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 4: IR %src.gep = getelementptr inbounds float, ptr %src, i64 %iv
; CHECK: Cost of 0 for VF 4: IR %l = load float, ptr %src.gep, align 4
; CHECK: Cost of 0 for VF 4: IR %c = fcmp olt float %l, 0.000000e+00
; CHECK: Cost of 1 for VF 4: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<%cmp.n>
; CHECK: LV: Selecting VF: 4.
;
entry:
br label %loop.header
loop.header:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
%src.gep = getelementptr inbounds float, ptr %src, i64 %iv
%l = load float, ptr %src.gep, align 4
%c = fcmp olt float %l, 0.000000e+00
br i1 %c, label %then, label %loop.latch
then:
%add = fadd float %l, 1.000000e+00
br label %loop.latch
loop.latch:
%p = phi float [ %l, %loop.header ], [ %add, %then ]
%dst.gep = getelementptr inbounds float, ptr %dst, i64 %iv
store float %p, ptr %dst.gep, align 4
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop.header
exit:
ret void
}
define void @blend_fcmp_uno_f32(ptr noalias %dst, ptr noalias %src, i64 %n) {
; CHECK-LABEL: 'blend_fcmp_uno_f32'
; CHECK: Cost of 0 for VF 2: forced scalar %src.gep = getelementptr inbounds float, ptr %src, i64 %iv
; CHECK: Cost of 0 for VF 2: forced scalar %dst.gep = getelementptr inbounds float, ptr %dst, i64 %iv
; CHECK: Cost of 0 for VF 2: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3:%[0-9]+]]>, ir<1>, vp<[[VP0:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: CLONE ir<%src.gep> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 3 for VF 2: WIDEN ir<%c> = fcmp uno ir<%l>, ir<0.000000e+00>
; CHECK: Cost of 1 for VF 2: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
; CHECK: Cost of 2 for VF 2: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 2: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 2: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
; CHECK: Cost of 1 for VF 2: WIDEN store vp<[[VP6]]>, ir<%p>
; CHECK: Cost of 0 for VF 2: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK: Cost of 1 for VF 2: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK: Cost of 0 for VF 2: vector loop backedge
; CHECK: Cost of 1 for VF 2: canonical IV increment
; CHECK: Cost of 0 for VF 2: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 2: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 2: IR %src.gep = getelementptr inbounds float, ptr %src, i64 %iv
; CHECK: Cost of 0 for VF 2: IR %l = load float, ptr %src.gep, align 4
; CHECK: Cost of 0 for VF 2: IR %c = fcmp uno float %l, 0.000000e+00
; CHECK: Cost of 1 for VF 2: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 2: EMIT branch-on-cond vp<%cmp.n>
; CHECK: Cost of 0 for VF 4: forced scalar %src.gep = getelementptr inbounds float, ptr %src, i64 %iv
; CHECK: Cost of 0 for VF 4: forced scalar %dst.gep = getelementptr inbounds float, ptr %dst, i64 %iv
; CHECK: Cost of 0 for VF 4: vp<[[VP4]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK: Cost of 0 for VF 4: CLONE ir<%src.gep> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP5]]> = vector-pointer inbounds float, ir<%src.gep>, ir<1>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%l> = load vp<[[VP5]]>
; CHECK: Cost of 3 for VF 4: WIDEN ir<%c> = fcmp uno ir<%l>, ir<0.000000e+00>
; CHECK: Cost of 1 for VF 4: WIDEN ir<%add> = fadd ir<%l>, ir<1.000000e+00>
; CHECK: Cost of 2 for VF 4: BLEND ir<%p> = ir<%l> ir<%add>/ir<%c>
; CHECK: Cost of 0 for VF 4: CLONE ir<%dst.gep> = getelementptr inbounds ir<%dst>, vp<[[VP4]]>
; CHECK: Cost of 0 for VF 4: vp<[[VP6]]> = vector-pointer inbounds float, ir<%dst.gep>, ir<1>
; CHECK: Cost of 1 for VF 4: WIDEN store vp<[[VP6]]>, ir<%p>
; CHECK: Cost of 0 for VF 4: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK: Cost of 1 for VF 4: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 4: vector loop backedge
; CHECK: Cost of 1 for VF 4: canonical IV increment
; CHECK: Cost of 0 for VF 4: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK: Cost of 0 for VF 4: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
; CHECK: Cost of 0 for VF 4: IR %src.gep = getelementptr inbounds float, ptr %src, i64 %iv
; CHECK: Cost of 0 for VF 4: IR %l = load float, ptr %src.gep, align 4
; CHECK: Cost of 0 for VF 4: IR %c = fcmp uno float %l, 0.000000e+00
; CHECK: Cost of 1 for VF 4: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK: Cost of 0 for VF 4: EMIT branch-on-cond vp<%cmp.n>
; CHECK: LV: Selecting VF: 4.
;
entry:
br label %loop.header
loop.header:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
%src.gep = getelementptr inbounds float, ptr %src, i64 %iv
%l = load float, ptr %src.gep, align 4
%c = fcmp uno float %l, 0.000000e+00
br i1 %c, label %then, label %loop.latch
then:
%add = fadd float %l, 1.000000e+00
br label %loop.latch
loop.latch:
%p = phi float [ %l, %loop.header ], [ %add, %then ]
%dst.gep = getelementptr inbounds float, ptr %dst, i64 %iv
store float %p, ptr %dst.gep, align 4
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop.header
exit:
ret void
}