test/Transforms/LoopVectorize/VPlan/compress-idioms.ll

; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:" --version 6
; RUN: opt -force-target-supports-masked-memory-ops -force-vector-width=4 -passes=loop-vectorize -vplan-print-after=printOptimizedVPlan -disable-output %s -S 2>&1 | FileCheck %s

define void @compress_store(ptr writeonly noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
; CHECK-LABEL: VPlan for loop in 'compress_store'
; CHECK:  VPlan 'Initial VPlan for VF={4},UF>=1' {
; CHECK-NEXT:  Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT:  Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
; CHECK-NEXT:  Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
; CHECK-NEXT:  Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<entry>:
; CHECK-NEXT:  Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT:  vector.ph:
; CHECK-NEXT:  Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT:  <x1> vector loop: {
; CHECK-NEXT:  vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT:    vector.body:
; CHECK-NEXT:      CONDITIONAL-INDUCTION-PHI ir<%idx> = phi ir<0>, vp<%conditional.step>
; CHECK-NEXT:      vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT:      CLONE ir<%src.ptr> = getelementptr inbounds ir<%src>, vp<[[VP4]]>
; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds i32, ir<%src.ptr>, ir<1>
; CHECK-NEXT:      WIDEN ir<%load.src> = load vp<[[VP5]]>, align 4
; CHECK-NEXT:      WIDEN ir<%cmp> = icmp slt ir<%load.src>, ir<%c>
; CHECK-NEXT:      CLONE ir<%dst.ptr> = getelementptr inbounds ir<%dst>, ir<%idx>
; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds i32, ir<%dst.ptr>, ir<1>
; CHECK-NEXT:      WIDEN-INTRINSIC vp<[[VP7:%[0-9]+]]> = call llvm.masked.compressstore(ir<%load.src>, vp<[[VP6]]>, ir<%cmp>) (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT:      EMIT vp<%handled.lanes> = num-active-lanes ir<%cmp>
; CHECK-NEXT:      EMIT vp<%conditional.step> = add ir<%idx>, vp<%handled.lanes>
; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT:    No successors
; CHECK-NEXT:  }
; CHECK-NEXT:  Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT:  middle.block:
; CHECK-NEXT:    EMIT vp<[[VP9:%[0-9]+]]> = extract-last-part vp<%conditional.step>
; CHECK-NEXT:    EMIT vp<[[VP10:%[0-9]+]]> = extract-last-lane vp<[[VP9]]>
; CHECK-NEXT:    EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK-NEXT:    EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT:  Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<exit>:
; CHECK-NEXT:  No successors
; CHECK-EMPTY:
; CHECK-NEXT:  scalar.ph:
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %iv
  %load.src = load i32, ptr %src.ptr, align 4
  %cmp = icmp slt i32 %load.src, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %idx
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %n
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}

define void @expand_load(ptr noalias %dst, ptr readonly %src, i32 %c, i64 %n) {
; CHECK-LABEL: VPlan for loop in 'expand_load'
; CHECK:  VPlan 'Initial VPlan for VF={4},UF>=1' {
; CHECK-NEXT:  Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT:  Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
; CHECK-NEXT:  Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
; CHECK-NEXT:  Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<entry>:
; CHECK-NEXT:  Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT:  vector.ph:
; CHECK-NEXT:  Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT:  <x1> vector loop: {
; CHECK-NEXT:  vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT:    vector.body:
; CHECK-NEXT:      CONDITIONAL-INDUCTION-PHI ir<%idx> = phi ir<0>, vp<%conditional.step>
; CHECK-NEXT:      vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT:      CLONE ir<%dst.ptr> = getelementptr ir<%dst>, vp<[[VP4]]>
; CHECK-NEXT:      vp<[[VP5:%[0-9]+]]> = vector-pointer i32, ir<%dst.ptr>, ir<1>
; CHECK-NEXT:      WIDEN ir<%load.dst> = load vp<[[VP5]]>, align 4
; CHECK-NEXT:      WIDEN ir<%cmp> = icmp slt ir<%load.dst>, ir<%c>
; CHECK-NEXT:      CLONE ir<%src.ptr> = getelementptr inbounds ir<%src>, ir<%idx>
; CHECK-NEXT:      vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds i32, ir<%src.ptr>, ir<1>
; CHECK-NEXT:      WIDEN-INTRINSIC vp<[[VP7:%[0-9]+]]> = call llvm.masked.expandload(vp<[[VP6]]>, ir<%cmp>, ir<poison>) (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT:      WIDEN store vp<[[VP5]]>, vp<[[VP7]]>, ir<%cmp>, align 4 (!vplan.execution.frequency 9223372036854775807 (50%, estimated))
; CHECK-NEXT:      EMIT vp<%handled.lanes> = num-active-lanes ir<%cmp>
; CHECK-NEXT:      EMIT vp<%conditional.step> = add ir<%idx>, vp<%handled.lanes>
; CHECK-NEXT:      EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT:      EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT:    No successors
; CHECK-NEXT:  }
; CHECK-NEXT:  Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT:  middle.block:
; CHECK-NEXT:    EMIT vp<[[VP9:%[0-9]+]]> = extract-last-part vp<%conditional.step>
; CHECK-NEXT:    EMIT vp<[[VP10:%[0-9]+]]> = extract-last-lane vp<[[VP9]]>
; CHECK-NEXT:    EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK-NEXT:    EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT:  Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT:  ir-bb<exit>:
; CHECK-NEXT:  No successors
; CHECK-EMPTY:
; CHECK-NEXT:  scalar.ph:
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
  %idx = phi i64 [ 0, %entry ], [ %idx.1, %for.inc ]
  %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %iv
  %load.dst = load i32, ptr %dst.ptr, align 4
  %cmp = icmp slt i32 %load.dst, %c
  br i1 %cmp, label %if.then, label %for.inc

if.then:
  %src.ptr = getelementptr inbounds i32, ptr %src, i64 %idx
  %load.src = load i32, ptr %src.ptr, align 4
  store i32 %load.src, ptr %dst.ptr, align 4
  %idx.next = add nsw i64 %idx, 1
  br label %for.inc

for.inc:
  %idx.1 = phi i64 [ %idx.next, %if.then ], [ %idx, %for.body ]
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %n
  br i1 %exitcond.not, label %exit, label %for.body

exit:
  ret void
}