llvm/test/Transforms/LoopVectorize/replicate-region-branch-costs.ll

; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "(Cost of .*(REPLICATE store|BRANCH-ON-MASK|unconditional branches from|profitable to scalarize))|(Precomputed costs for VF)|(Cost for VF)" --filter-out-after "LV: (Selecting|Using user) VF" --version 6
; REQUIRES: asserts
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -vectorize-num-stores-pred=4 -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck --check-prefix=CHECK %s
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -vectorize-num-stores-pred=4 -tail-folding-policy=prefer-fold-tail -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck --check-prefix=FOLD %s

define void @predicated_store(ptr noalias %dst, i64 %x, i64 %n) {
; CHECK-LABEL: 'predicated_store'
; CHECK:  Cost of 2 for VF 4: profitable to scalarize store i32 1, ptr %gep.dst, align 4
; CHECK:  Precomputed costs for VF 4: 2
; CHECK:  Cost of 4 for VF 4: BRANCH-ON-MASK ir<%c> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK:  Cost of 0 for VF 4: REPLICATE store ir<1>, ir<%gep.dst>
; CHECK:  Cost of 2 for VF 4: unconditional branches from pred.store.if
; CHECK:  Cost for VF 4: 14.5 (Estimated cost per lane: 3.5)
;
; FOLD-LABEL: 'predicated_store'
; FOLD:  Cost of 2 for VF 4: profitable to scalarize store i32 1, ptr %gep.dst, align 4
; FOLD:  Precomputed costs for VF 4: 2
; FOLD:  Cost of 4 for VF 4: BRANCH-ON-MASK vp<[[VP8:%[0-9]+]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; FOLD:  Cost of 0 for VF 4: REPLICATE store ir<1>, ir<%gep.dst>
; FOLD:  Cost of 2 for VF 4: unconditional branches from pred.store.if
; FOLD:  Cost for VF 4: 15.5 (Estimated cost per lane: 3.75)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
  %c = icmp ult i64 %iv, %x
  br i1 %c, label %then, label %latch

then:
  %gep.dst = getelementptr i32, ptr %dst, i64 %iv
  store i32 1, ptr %gep.dst
  br label %latch

latch:
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

define void @two_predicated_stores(ptr noalias %dst.1, ptr noalias %dst.2, i64 %x, i64 %y, i64 %n) {
; CHECK-LABEL: 'two_predicated_stores'
; CHECK:  Cost of 2 for VF 4: profitable to scalarize store i32 1, ptr %gep.dst.1, align 4
; CHECK:  Cost of 2 for VF 4: profitable to scalarize store i32 2, ptr %gep.dst.2, align 4
; CHECK:  Precomputed costs for VF 4: 4
; CHECK:  Cost of 4 for VF 4: BRANCH-ON-MASK ir<%c.1> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK:  Cost of 0 for VF 4: REPLICATE store ir<1>, ir<%gep.dst.1>
; CHECK:  Cost of 2 for VF 4: unconditional branches from pred.store.if
; CHECK:  Cost of 4 for VF 4: BRANCH-ON-MASK ir<%c.2> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK:  Cost of 0 for VF 4: REPLICATE store ir<2>, ir<%gep.dst.2>
; CHECK:  Cost of 2 for VF 4: unconditional branches from pred.store.if
; CHECK:  Cost for VF 4: 24 (Estimated cost per lane: 6)
;
; FOLD-LABEL: 'two_predicated_stores'
; FOLD:  Cost of 2 for VF 4: profitable to scalarize store i32 1, ptr %gep.dst.1, align 4
; FOLD:  Cost of 2 for VF 4: profitable to scalarize store i32 2, ptr %gep.dst.2, align 4
; FOLD:  Precomputed costs for VF 4: 4
; FOLD:  Cost of 4 for VF 4: BRANCH-ON-MASK vp<[[VP9:%[0-9]+]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; FOLD:  Cost of 0 for VF 4: REPLICATE store ir<1>, ir<%gep.dst.1>
; FOLD:  Cost of 2 for VF 4: unconditional branches from pred.store.if
; FOLD:  Cost of 4 for VF 4: BRANCH-ON-MASK vp<[[VP10:%[0-9]+]]> (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; FOLD:  Cost of 0 for VF 4: REPLICATE store ir<2>, ir<%gep.dst.2>
; FOLD:  Cost of 2 for VF 4: unconditional branches from pred.store.if
; FOLD:  Cost for VF 4: 25 (Estimated cost per lane: 6.25)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
  %c.1 = icmp ult i64 %iv, %x
  br i1 %c.1, label %then.1, label %check.2

then.1:
  %gep.dst.1 = getelementptr i32, ptr %dst.1, i64 %iv
  store i32 1, ptr %gep.dst.1
  br label %check.2

check.2:
  %c.2 = icmp ugt i64 %iv, %y
  br i1 %c.2, label %then.2, label %latch

then.2:
  %gep.dst.2 = getelementptr i32, ptr %dst.2, i64 %iv
  store i32 2, ptr %gep.dst.2
  br label %latch

latch:
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; The replicate region is only predicated by the header mask with
; tail-folding.
define void @header_mask_only(ptr noalias %dst, i64 %n) {
; CHECK-LABEL: 'header_mask_only'
; CHECK:  Precomputed costs for VF 4: 0
; CHECK:  Cost for VF 4: 4 (Estimated cost per lane: 1)
;
; FOLD-LABEL: 'header_mask_only'
; FOLD:  Cost of 4 for VF 4: profitable to scalarize store i8 0, ptr %gep, align 1
; FOLD:  Precomputed costs for VF 4: 4
; FOLD:  Cost of 4 for VF 4: BRANCH-ON-MASK vp<[[VP7:%[0-9]+]]>
; FOLD:  Cost of 0 for VF 4: REPLICATE store ir<0>, ir<%gep>
; FOLD:  Cost of 4 for VF 4: unconditional branches from pred.store.if
; FOLD:  Cost for VF 4: 17 (Estimated cost per lane: 4.25)
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %gep = getelementptr i8, ptr %dst, i64 %iv
  store i8 0, ptr %gep
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}