llvm/test/Transforms/LoopVectorize/VPlan/memory-checks.ll

; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "vector.body:" --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -disable-output -vplan-print-after="printFinalVPlan" -vplan-print-metadata=false %s 2>&1 | FileCheck %s

define void @three_groups_shared_bounds(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-LABEL: VPlan for loop in 'three_groups_shared_bounds'
; CHECK:  VPlan 'Final VPlan for VF={4},UF={1}' {
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<entry>:
; CHECK-NEXT:    EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = call i64 @llvm.umax(ir<%n>, ir<1>)
; CHECK-NEXT:    EMIT vp<%min.iters.check> = icmp ult vp<[[VP2]]>, ir<4>
; CHECK-NEXT:    EMIT branch-on-cond vp<%min.iters.check>
; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, ir-bb<vector.memcheck>
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<vector.memcheck>:
; CHECK-NEXT:    IR   %umax = call i64 @llvm.umax.i64(i64 %n, i64 1)
; CHECK-NEXT:    IR   %0 = shl i64 %umax, 2
; CHECK-NEXT:    IR   %scevgep = getelementptr i8, ptr %c, i64 %0
; CHECK-NEXT:    IR   %scevgep1 = getelementptr i8, ptr %a, i64 %0
; CHECK-NEXT:    IR   %scevgep2 = getelementptr i8, ptr %b, i64 %0
; CHECK-NEXT:    IR   %bound0 = icmp ult ptr %c, %scevgep1
; CHECK-NEXT:    IR   %bound1 = icmp ult ptr %a, %scevgep
; CHECK-NEXT:    IR   %found.conflict = and i1 %bound0, %bound1
; CHECK-NEXT:    IR   %bound03 = icmp ult ptr %c, %scevgep2
; CHECK-NEXT:    IR   %bound14 = icmp ult ptr %b, %scevgep
; CHECK-NEXT:    IR   %found.conflict5 = and i1 %bound03, %bound14
; CHECK-NEXT:    IR   %conflict.rdx = or i1 %found.conflict, %found.conflict5
; CHECK-NEXT:    IR   %bound06 = icmp ult ptr %a, %scevgep2
; CHECK-NEXT:    IR   %bound17 = icmp ult ptr %b, %scevgep1
; CHECK-NEXT:    IR   %found.conflict8 = and i1 %bound06, %bound17
; CHECK-NEXT:    IR   %conflict.rdx9 = or i1 %conflict.rdx, %found.conflict8
; CHECK-NEXT:    EMIT branch-on-cond ir<%conflict.rdx9>
; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT:  vector.ph:
; CHECK-NEXT:    EMIT vp<[[VP5:%[0-9]+]]> = and vp<[[VP2]]>, ir<3>
; CHECK-NEXT:    EMIT vp<%n.vec> = sub vp<[[VP2]]>, vp<[[VP5]]>
; CHECK-NEXT:  Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT:  vector.body:
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %gep.a.read = getelementptr inbounds i32, ptr %a, i64 %iv
  %load.a = load i32, ptr %gep.a.read, align 4
  %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
  %load.b = load i32, ptr %gep.b, align 4
  %sum = add i32 %load.a, %load.b
  %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
  store i32 %sum, ptr %gep.c, align 4
  store i32 %load.b, ptr %gep.a.read, align 4
  %iv.next = add nuw nsw i64 %iv, 1
  %cmp = icmp ult i64 %iv.next, %n
  br i1 %cmp, label %loop, label %exit

exit:
  ret void
}

define void @ptr_minmax_bounds(ptr %a, ptr %b, i64 %n, i64 %s, i64 %t) {
; CHECK-LABEL: VPlan for loop in 'ptr_minmax_bounds'
; CHECK:  VPlan 'Final VPlan for VF={4},UF={1}' {
; CHECK-NEXT:  Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<entry>:
; CHECK-NEXT:    IR   %step = mul i64 %s, %t
; CHECK-NEXT:    EMIT vp<%min.iters.check> = icmp ult ir<%n>, ir<4>
; CHECK-NEXT:    EMIT branch-on-cond vp<%min.iters.check>
; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, ir-bb<vector.memcheck>
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<vector.memcheck>:
; CHECK-NEXT:    IR   %0 = shl i64 %n, 2
; CHECK-NEXT:    IR   %scevgep = getelementptr i8, ptr %b, i64 %0
; CHECK-NEXT:    IR   %1 = mul i64 %t, %s
; CHECK-NEXT:    IR   %2 = add i64 %n, -1
; CHECK-NEXT:    IR   %3 = mul i64 %1, %2
; CHECK-NEXT:    IR   %4 = shl i64 %3, 2
; CHECK-NEXT:    IR   %scevgep1 = getelementptr i8, ptr %a, i64 %4
; CHECK-NEXT:    IR   %5 = icmp ult ptr %a, %scevgep1
; CHECK-NEXT:    IR   %umin = select i1 %5, ptr %a, ptr %scevgep1
; CHECK-NEXT:    IR   %6 = icmp ugt ptr %a, %scevgep1
; CHECK-NEXT:    IR   %umax = select i1 %6, ptr %a, ptr %scevgep1
; CHECK-NEXT:    IR   %scevgep2 = getelementptr i8, ptr %umax, i64 4
; CHECK-NEXT:    IR   %bound0 = icmp ult ptr %b, %scevgep2
; CHECK-NEXT:    IR   %bound1 = icmp ult ptr %umin, %scevgep
; CHECK-NEXT:    IR   %found.conflict = and i1 %bound0, %bound1
; CHECK-NEXT:    EMIT branch-on-cond ir<%found.conflict>
; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT:  vector.ph:
; CHECK-NEXT:    EMIT vp<[[VP4:%[0-9]+]]> = and ir<%n>, ir<3>
; CHECK-NEXT:    EMIT vp<%n.vec> = sub ir<%n>, vp<[[VP4]]>
; CHECK-NEXT:    EMIT vp<[[VP5:%[0-9]+]]> = broadcast ir<%step>
; CHECK-NEXT:    EMIT vp<[[VP6:%[0-9]+]]> = step-vector i64
; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = broadcast ir<4>
; CHECK-NEXT:  Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT:  vector.body:
;
entry:
  %step = mul i64 %s, %t
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %idx = mul i64 %iv, %step
  %gep.a = getelementptr inbounds i32, ptr %a, i64 %idx
  %l = load i32, ptr %gep.a, align 4
  %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
  store i32 %l, ptr %gep.b, align 4
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, %n
  br i1 %ec, label %exit, label %loop

exit:
  ret void
}

define void @diff_check(ptr %a, ptr %b, ptr %c, i64 %n) {
; CHECK-LABEL: VPlan for loop in 'diff_check'
; CHECK:  VPlan 'Final VPlan for VF={4},UF={1}' {
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<entry>:
; CHECK-NEXT:    IR   %b3 = ptrtoaddr ptr %b to i64
; CHECK-NEXT:    IR   %a2 = ptrtoaddr ptr %a to i64
; CHECK-NEXT:    IR   %c1 = ptrtoaddr ptr %c to i64
; CHECK-NEXT:    EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = call i64 @llvm.umax(ir<%n>, ir<1>)
; CHECK-NEXT:    EMIT vp<%min.iters.check> = icmp ult vp<[[VP2]]>, ir<4>
; CHECK-NEXT:    EMIT branch-on-cond vp<%min.iters.check>
; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, ir-bb<vector.memcheck>
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<vector.memcheck>:
; CHECK-NEXT:    IR   %0 = sub i64 %c1, %a2
; CHECK-NEXT:    IR   %1 = sub i64 %0, 1
; CHECK-NEXT:    IR   %diff.check = icmp ult i64 %1, 15
; CHECK-NEXT:    IR   %2 = sub i64 %c1, %b3
; CHECK-NEXT:    IR   %3 = sub i64 %2, 1
; CHECK-NEXT:    IR   %diff.check4 = icmp ult i64 %3, 15
; CHECK-NEXT:    IR   %conflict.rdx = or i1 %diff.check, %diff.check4
; CHECK-NEXT:    EMIT branch-on-cond ir<%conflict.rdx>
; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT:  vector.ph:
; CHECK-NEXT:    EMIT vp<[[VP5:%[0-9]+]]> = and vp<[[VP2]]>, ir<3>
; CHECK-NEXT:    EMIT vp<%n.vec> = sub vp<[[VP2]]>, vp<[[VP5]]>
; CHECK-NEXT:  Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT:  vector.body:
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
  %load.a = load i32, ptr %gep.a, align 4
  %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
  %load.b = load i32, ptr %gep.b, align 4
  %sum = add i32 %load.a, %load.b
  %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
  store i32 %sum, ptr %gep.c, align 4
  %iv.next = add nuw nsw i64 %iv, 1
  %cmp = icmp ult i64 %iv.next, %n
  br i1 %cmp, label %loop, label %exit

exit:
  ret void
}

; One of the bounds of the runtime check is an AddRec of the preceding
; sibling loop.
define void @bound_is_addrec_of_sibling_loop(ptr %a, ptr %b, i64 %n, i64 %d, i1 %cond) {
; CHECK-LABEL: VPlan for loop in 'bound_is_addrec_of_sibling_loop'
; CHECK:  VPlan 'Final VPlan for VF={4},UF={1}' {
; CHECK-NEXT:  Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<ph.2>:
; CHECK-NEXT:    IR   %iv.1.lcssa = phi i64 [ %iv.1, %loop.1 ]
; CHECK-NEXT:    EMIT vp<%min.iters.check> = icmp ult ir<%n>, ir<4>
; CHECK-NEXT:    EMIT branch-on-cond vp<%min.iters.check>
; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, ir-bb<vector.scevcheck>
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<vector.scevcheck>:
; CHECK-NEXT:    IR   %0 = add i64 %n, -1
; CHECK-NEXT:    IR   %mul = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 8, i64 %0)
; CHECK-NEXT:    IR   %mul.result = extractvalue { i64, i1 } %mul, 0
; CHECK-NEXT:    IR   %mul.overflow = extractvalue { i64, i1 } %mul, 1
; CHECK-NEXT:    IR   %1 = getelementptr i8, ptr %a, i64 %mul.result
; CHECK-NEXT:    IR   %2 = icmp ult ptr %1, %a
; CHECK-NEXT:    IR   %3 = or i1 %2, %mul.overflow
; CHECK-NEXT:    EMIT branch-on-cond ir<%3>
; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, ir-bb<vector.memcheck>
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<vector.memcheck>:
; CHECK-NEXT:    IR   %4 = udiv i64 %n, %d
; CHECK-NEXT:    IR   %5 = shl i64 %4, 2
; CHECK-NEXT:    IR   %6 = shl i64 %iv.1, 2
; CHECK-NEXT:    IR   %7 = add i64 %6, %5
; CHECK-NEXT:    IR   %scevgep = getelementptr i8, ptr %b, i64 %7
; CHECK-NEXT:    IR   %8 = shl i64 %n, 2
; CHECK-NEXT:    IR   %9 = add i64 %8, %5
; CHECK-NEXT:    IR   %10 = add i64 %6, %9
; CHECK-NEXT:    IR   %scevgep1 = getelementptr i8, ptr %b, i64 %10
; CHECK-NEXT:    IR   %11 = shl i64 %n, 3
; CHECK-NEXT:    IR   %12 = add i64 %11, -4
; CHECK-NEXT:    IR   %scevgep2 = getelementptr i8, ptr %a, i64 %12
; CHECK-NEXT:    IR   %bound0 = icmp ult ptr %scevgep, %scevgep2
; CHECK-NEXT:    IR   %bound1 = icmp ult ptr %a, %scevgep1
; CHECK-NEXT:    IR   %found.conflict = and i1 %bound0, %bound1
; CHECK-NEXT:    EMIT branch-on-cond ir<%found.conflict>
; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT:  vector.ph:
; CHECK-NEXT:    EMIT vp<[[VP5:%[0-9]+]]> = and ir<%n>, ir<3>
; CHECK-NEXT:    EMIT vp<%n.vec> = sub ir<%n>, vp<[[VP5]]>
; CHECK-NEXT:    EMIT vp<[[VP6:%[0-9]+]]> = broadcast ir<%d>
; CHECK-NEXT:    EMIT vp<[[VP7:%[0-9]+]]> = broadcast ir<%n>
; CHECK-NEXT:    EMIT vp<[[VP8:%[0-9]+]]> = broadcast ir<%cond>
; CHECK-NEXT:    WIDEN-INTRINSIC vp<[[VP9:%[0-9]+]]> = call llvm.masked.udiv(vp<[[VP7]]>, vp<[[VP6]]>, vp<[[VP8]]>)
; CHECK-NEXT:    EMIT vp<[[VP10:%[0-9]+]]> = broadcast ir<%iv.1.lcssa>
; CHECK-NEXT:    EMIT vp<[[VP11:%[0-9]+]]> = step-vector i64
; CHECK-NEXT:    EMIT vp<[[VP12:%[0-9]+]]> = broadcast ir<4>
; CHECK-NEXT:  Successor(s): vector.body
; CHECK-EMPTY:
; CHECK-NEXT:  vector.body:
;
entry:
  br label %loop.1

loop.1:
  %iv.1 = phi i64 [ 0, %entry ], [ %iv.1.next, %loop.1 ]
  call void @use(i64 %iv.1)
  %iv.1.next = add nuw nsw i64 %iv.1, 1
  %ec.1 = icmp eq i64 %iv.1.next, %n
  br i1 %ec.1, label %ph.2, label %loop.1

ph.2:
  br label %loop.2

loop.2:
  %iv.2 = phi i64 [ 0, %ph.2 ], [ %iv.2.next, %latch ]
  br i1 %cond, label %then, label %latch

then:
  %q = udiv i64 %n, %d
  %idx.a = shl nuw nsw i64 %iv.2, 1
  %gep.a = getelementptr inbounds i32, ptr %a, i64 %idx.a
  %l = load i32, ptr %gep.a, align 4
  %off = add nuw nsw i64 %iv.2, %iv.1
  %idx.b = add nuw nsw i64 %off, %q
  %gep.b = getelementptr inbounds i32, ptr %b, i64 %idx.b
  store i32 %l, ptr %gep.b, align 4
  br label %latch

latch:
  %iv.2.next = add nuw nsw i64 %iv.2, 1
  %ec.2 = icmp eq i64 %iv.2.next, %n
  br i1 %ec.2, label %exit, label %loop.2

exit:
  ret void
}

declare void @use(i64)