llvm/test/Transforms/LoopVectorize/replicate-region-cost-probability.ll

; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of.*REPLICATE" --version 6
; REQUIRES: asserts
; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -disable-output -debug-only=loop-vectorize %s 2>&1 | FileCheck %s

; The predicate is taken 1 in 4 iterations (reciprocal 4): cost 10 / 4 -> 2.5.
define void @predicated_udiv_p25(ptr %a, i32 %n) {
; CHECK-LABEL: 'predicated_udiv_p25'
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 2.5 for VF 2: REPLICATE ir<%d> = udiv ir<%val>, ir<7>
; CHECK:  Cost of 0 for VF 2: REPLICATE store ir<%d>, ir<%gep>.1
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP6]]>
; CHECK:  Cost of 2.5 for VF 2: REPLICATE ir<%d> = udiv ir<%val>, ir<7>
; CHECK:  Cost of 0 for VF 2: REPLICATE store ir<%d>, ir<%gep>.1
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %cmp = icmp sgt i32 %val, 0
  br i1 %cmp, label %if.then, label %latch, !prof !1

if.then:
  %d = udiv i32 %val, 7
  store i32 %d, ptr %gep, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

; The predicate is taken 1 in 8 iterations (reciprocal 8): cost 10 / 8 -> 1.25.
; A less likely predicate makes the predicated udiv look cheaper than at 1/4.
define void @predicated_udiv_p12(ptr %a, i32 %n) {
; CHECK-LABEL: 'predicated_udiv_p12'
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 1.25 for VF 2: REPLICATE ir<%d> = udiv ir<%val>, ir<7>
; CHECK:  Cost of 0 for VF 2: REPLICATE store ir<%d>, ir<%gep>.1
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP6]]>
; CHECK:  Cost of 1.25 for VF 2: REPLICATE ir<%d> = udiv ir<%val>, ir<7>
; CHECK:  Cost of 0 for VF 2: REPLICATE store ir<%d>, ir<%gep>.1
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %cmp = icmp sgt i32 %val, 0
  br i1 %cmp, label %if.then, label %latch, !prof !2

if.then:
  %d = udiv i32 %val, 7
  store i32 %d, ptr %gep, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

; Without profile data the cost model is still self-contained: the recorded
; reciprocal comes from branch weights estimated by BranchProbabilityInfo. For
; this comparison against zero that heuristic favors the predicate at 5/8
; (round(1/0.625) = 2), so cost 10 / 2 -> 5.
define void @predicated_udiv_no_profile(ptr %a, i32 %n) {
; CHECK-LABEL: 'predicated_udiv_no_profile'
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 5 for VF 2: REPLICATE ir<%d> = udiv ir<%val>, ir<7>
; CHECK:  Cost of 0 for VF 2: REPLICATE store ir<%d>, ir<%gep>.1
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP6]]>
; CHECK:  Cost of 5 for VF 2: REPLICATE ir<%d> = udiv ir<%val>, ir<7>
; CHECK:  Cost of 0 for VF 2: REPLICATE store ir<%d>, ir<%gep>.1
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %cmp = icmp sgt i32 %val, 0
  br i1 %cmp, label %if.then, label %latch

if.then:
  %d = udiv i32 %val, 7
  store i32 %d, ptr %gep, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop

exit:
  ret void
}

; Two adjacent predicated blocks guarded by the same condition are merged into a
; single replicate region. 2 branches with same condition have different probabilities.
define void @merged_regions_use_conservative_probability(ptr noalias %a, ptr noalias %b, i32 %n, i32 %x, i32 %y) {
; CHECK-LABEL: 'merged_regions_use_conservative_probability'
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP7:%[0-9]+]]>
; CHECK:  Cost of 3000000 for VF 2: REPLICATE store vp<[[VP6:%[0-9]+]]>, ir<%gep>.1
; CHECK:  Cost of 2.5 for VF 2: REPLICATE ir<%d2> = sdiv ir<%val>, ir<%y>
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<[[VP7]]>
; CHECK:  Cost of 3000000 for VF 2: REPLICATE store ir<%d2>, ir<%gep.b>
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP7]]>
; CHECK:  Cost of 3000000 for VF 2: REPLICATE store vp<[[VP6]]>, ir<%gep>.1
; CHECK:  Cost of 2.5 for VF 2: REPLICATE ir<%d2> = sdiv ir<%val>, ir<%y>
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<[[VP7]]>
; CHECK:  Cost of 3000000 for VF 2: REPLICATE store ir<%d2>, ir<%gep.b>
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %cmp = icmp sgt i32 %val, 0
  br i1 %cmp, label %then1, label %bb1, !prof !1

then1:
  %d1 = sdiv i32 %val, %x
  store i32 %d1, ptr %gep, align 4
  br label %bb1

bb1:
  br i1 %cmp, label %then2, label %latch, !prof !3

then2:
  %d2 = sdiv i32 %val, %y
  %gep.b = getelementptr inbounds i32, ptr %b, i32 %iv
  store i32 %d2, ptr %gep.b, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

define void @sunk_recipe_uses_region_probability(ptr noalias %a, ptr noalias %b, i32 %n, i32 %x) {
; CHECK-LABEL: 'sunk_recipe_uses_region_probability'
; CHECK:  Cost of 2.5 for VF 2: REPLICATE ir<%rem> = srem ir<%val>, ir<%x>
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: REPLICATE store ir<%rem>, ir<%gep.b>
; CHECK:  Cost of 2.5 for VF 2: REPLICATE ir<%rem> = srem ir<%val>, ir<%x>
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<[[VP6]]>
; CHECK:  Cost of 0 for VF 2: REPLICATE store ir<%rem>, ir<%gep.b>
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %cmp = icmp sgt i32 %val, 0
  br i1 %cmp, label %if.then, label %latch, !prof !1

if.then:
  %rem = srem i32 %val, %x
  %gep.b = getelementptr inbounds i32, ptr %b, i32 %iv
  store i32 %rem, ptr %gep.b, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

!0 = !{!"branch_weights", i32 1, i32 1000}
!1 = !{!"branch_weights", i32 1, i32 3}
!2 = !{!"branch_weights", i32 1, i32 7}
!3 = !{!"branch_weights", i32 1, i32 99}