llvm/test/Transforms/LoopVectorize/VPlan/cse-loads-masked.ll

; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block" --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
; RUN:     -force-target-supports-masked-memory-ops -disable-output \
; RUN:     -vplan-print-after="printFinalVPlan$" %s 2>&1 | FileCheck %s
; REQUIRES: asserts

; Tests CSE of masked widened loads and mixed masked/regular loads.

; Two masked loads from the same address under the same condition are CSE'd.
define void @two_masked_same_mask(ptr noalias %a, ptr noalias %b, ptr noalias %cond) {
; CHECK-LABEL: VPlan for loop in 'two_masked_same_mask'
; CHECK:  VPlan 'Final VPlan for VF={4},UF={1}' {
; CHECK-NEXT:  Live-in ir<1024> = vector-trip-count
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<entry>:
; CHECK-NEXT:  Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT:  vector.ph:
; CHECK-NEXT:  Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT:  vector.body:
; CHECK-NEXT:    EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
; CHECK-NEXT:    CLONE ir<%gep.c> = getelementptr inbounds ir<%cond>, vp<%index>
; CHECK-NEXT:    WIDEN ir<%c> = load ir<%gep.c>
; CHECK-NEXT:    WIDEN ir<%cmp> = icmp ne ir<%c>, ir<0>
; CHECK-NEXT:    CLONE ir<%gep.a> = getelementptr ir<%a>, vp<%index>
; CHECK-NEXT:    WIDEN ir<%x> = load ir<%gep.a>, ir<%cmp>
; CHECK-NEXT:    WIDEN ir<%s> = add ir<%x>, ir<%x>
; CHECK-NEXT:    EMIT vp<%predphi> = select ir<%cmp>, ir<%s>, ir<0>
; CHECK-NEXT:    CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
; CHECK-NEXT:    WIDEN store ir<%gep.b>, vp<%predphi>
; CHECK-NEXT:    EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
; CHECK-NEXT:    EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
; CHECK-NEXT:    EMIT branch-on-cond vp<[[VP1]]>
; CHECK-NEXT:  Successor(s): middle.block, vector.body
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
  %gep.c = getelementptr inbounds i32, ptr %cond, i64 %iv
  %c = load i32, ptr %gep.c, align 4
  %cmp = icmp ne i32 %c, 0
  br i1 %cmp, label %if, label %latch

if:
  %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
  %x = load i32, ptr %gep.a, align 4
  %y = load i32, ptr %gep.a, align 4
  %s = add i32 %x, %y
  br label %latch

latch:
  %m = phi i32 [ 0, %loop ], [ %s, %if ]
  %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
  store i32 %m, ptr %gep.b, align 4
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; Masked loads from the same address under different conditions are not CSE'd.
define void @two_masked_diff_mask(ptr noalias %a, ptr noalias %b, ptr noalias %c1, ptr noalias %c2) {
; CHECK-LABEL: VPlan for loop in 'two_masked_diff_mask'
; CHECK:  VPlan 'Final VPlan for VF={4},UF={1}' {
; CHECK-NEXT:  Live-in ir<1024> = vector-trip-count
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<entry>:
; CHECK-NEXT:  Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT:  vector.ph:
; CHECK-NEXT:  Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT:  vector.body:
; CHECK-NEXT:    EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
; CHECK-NEXT:    CLONE ir<%gep.a> = getelementptr ir<%a>, vp<%index>
; CHECK-NEXT:    CLONE ir<%gp1> = getelementptr inbounds ir<%c1>, vp<%index>
; CHECK-NEXT:    WIDEN ir<%v1> = load ir<%gp1>
; CHECK-NEXT:    WIDEN ir<%cmp1> = icmp ne ir<%v1>, ir<0>
; CHECK-NEXT:    WIDEN ir<%x> = load ir<%gep.a>, ir<%cmp1>
; CHECK-NEXT:    EMIT vp<%predphi> = select ir<%cmp1>, ir<%x>, ir<0>
; CHECK-NEXT:    CLONE ir<%gp2> = getelementptr inbounds ir<%c2>, vp<%index>
; CHECK-NEXT:    WIDEN ir<%v2> = load ir<%gp2>
; CHECK-NEXT:    WIDEN ir<%cmp2> = icmp ne ir<%v2>, ir<0>
; CHECK-NEXT:    WIDEN ir<%y> = load ir<%gep.a>, ir<%cmp2>
; CHECK-NEXT:    EMIT vp<%predphi>.1 = select ir<%cmp2>, ir<%y>, ir<0>
; CHECK-NEXT:    WIDEN ir<%sum> = add vp<%predphi>, vp<%predphi>.1
; CHECK-NEXT:    CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
; CHECK-NEXT:    WIDEN store ir<%gep.b>, ir<%sum>
; CHECK-NEXT:    EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
; CHECK-NEXT:    EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
; CHECK-NEXT:    EMIT branch-on-cond vp<[[VP1]]>
; CHECK-NEXT:  Successor(s): middle.block, vector.body
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch2 ]
  %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
  %gp1 = getelementptr inbounds i32, ptr %c1, i64 %iv
  %v1 = load i32, ptr %gp1, align 4
  %cmp1 = icmp ne i32 %v1, 0
  br i1 %cmp1, label %if1, label %latch1

if1:
  %x = load i32, ptr %gep.a, align 4
  br label %latch1

latch1:
  %mx = phi i32 [ 0, %loop ], [ %x, %if1 ]
  %gp2 = getelementptr inbounds i32, ptr %c2, i64 %iv
  %v2 = load i32, ptr %gp2, align 4
  %cmp2 = icmp ne i32 %v2, 0
  br i1 %cmp2, label %if2, label %latch2

if2:
  %y = load i32, ptr %gep.a, align 4
  br label %latch2

latch2:
  %my = phi i32 [ 0, %latch1 ], [ %y, %if2 ]
  %sum = add i32 %mx, %my
  %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
  store i32 %sum, ptr %gep.b, align 4
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

; A duplicated regular load and a duplicated masked load are each CSE'd
; independently.
define void @masked_and_regular_mix(ptr noalias %a, ptr noalias %b, ptr noalias %cond, ptr noalias %out) {
; CHECK-LABEL: VPlan for loop in 'masked_and_regular_mix'
; CHECK:  VPlan 'Final VPlan for VF={4},UF={1}' {
; CHECK-NEXT:  Live-in ir<1024> = vector-trip-count
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<entry>:
; CHECK-NEXT:  Successor(s): vector.ph
; CHECK-EMPTY:
; CHECK-NEXT:  vector.ph:
; CHECK-NEXT:  Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT:  vector.body:
; CHECK-NEXT:    EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
; CHECK-NEXT:    CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<%index>
; CHECK-NEXT:    WIDEN ir<%r1> = load ir<%gep.a>
; CHECK-NEXT:    CLONE ir<%gep.c> = getelementptr inbounds ir<%cond>, vp<%index>
; CHECK-NEXT:    WIDEN ir<%c> = load ir<%gep.c>
; CHECK-NEXT:    WIDEN ir<%cmp> = icmp ne ir<%c>, ir<0>
; CHECK-NEXT:    CLONE ir<%gep.b> = getelementptr ir<%b>, vp<%index>
; CHECK-NEXT:    WIDEN ir<%m1> = load ir<%gep.b>, ir<%cmp>
; CHECK-NEXT:    WIDEN ir<%ms> = add ir<%m1>, ir<%m1>
; CHECK-NEXT:    EMIT vp<%predphi> = select ir<%cmp>, ir<%ms>, ir<0>
; CHECK-NEXT:    WIDEN ir<%rr> = add ir<%r1>, ir<%r1>
; CHECK-NEXT:    WIDEN ir<%sum> = add ir<%rr>, vp<%predphi>
; CHECK-NEXT:    CLONE ir<%gep.o> = getelementptr inbounds ir<%out>, vp<%index>
; CHECK-NEXT:    WIDEN store ir<%gep.o>, ir<%sum>
; CHECK-NEXT:    EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
; CHECK-NEXT:    EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
; CHECK-NEXT:    EMIT branch-on-cond vp<[[VP1]]>
; CHECK-NEXT:  Successor(s): middle.block, vector.body
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
  %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
  %r1 = load i32, ptr %gep.a, align 4
  %r2 = load i32, ptr %gep.a, align 4
  %gep.c = getelementptr inbounds i32, ptr %cond, i64 %iv
  %c = load i32, ptr %gep.c, align 4
  %cmp = icmp ne i32 %c, 0
  br i1 %cmp, label %if, label %latch

if:
  %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
  %m1 = load i32, ptr %gep.b, align 4
  %m2 = load i32, ptr %gep.b, align 4
  %ms = add i32 %m1, %m2
  br label %latch

latch:
  %mm = phi i32 [ 0, %loop ], [ %ms, %if ]
  %rr = add i32 %r1, %r2
  %sum = add i32 %rr, %mm
  %gep.o = getelementptr inbounds i32, ptr %out, i64 %iv
  store i32 %sum, ptr %gep.o, align 4
  %iv.next = add i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}