llvm/test/Transforms/LoopVectorize/AArch64/memory-widening-debug.ll

; REQUIRES: asserts
; RUN: opt -passes=loop-vectorize -force-vector-width=4 \
; RUN:   -force-vector-interleave=1 -tail-folding-policy=dont-fold-tail \
; RUN:   -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck %s --check-prefix=FIXED
; RUN: opt -passes=loop-vectorize -mattr=+sve -force-vector-width="vscale x 2" \
; RUN:   -force-vector-interleave=1 -tail-folding-policy=dont-fold-tail \
; RUN:   -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck %s --check-prefix=SVE

target triple = "aarch64"

@a = common global [2048 x i32] zeroinitializer, align 16
@b = common global [2048 x i32] zeroinitializer, align 16
@c = common global [2048 x i32] zeroinitializer, align 16
@d = common global [2048 x i32] zeroinitializer, align 16

define void @uniform_load(ptr noalias %dst, ptr noalias readonly %src, i64 %n) {
; FIXED-LABEL: 'uniform_load'
; FIXED:  LV: Memory widening: calculating best strategy for %val = load i32, ptr %src, align 4
; FIXED:  LV: Memory widening: uniform memory op has GatherScatterCost = Invalid, ScalarizationCost = 3
; FIXED:  LV: Setting widening decision to Scalarize for VF 4 and instruction: %val = load i32, ptr %src, align 4
; FIXED:  LV: Memory widening: calculating best strategy for store i32 %val, ptr %arrayidx, align 4
; FIXED:  LV: Memory widening: can be widened normally with cost 1
; FIXED:  LV: Setting widening decision to Widen for VF 4 and instruction: store i32 %val, ptr %arrayidx, align 4
;
; SVE-LABEL: 'uniform_load'
; SVE:  LV: Memory widening: calculating best strategy for %val = load i32, ptr %src, align 4
; SVE:  LV: Memory widening: uniform memory op has GatherScatterCost = 21, ScalarizationCost = 3
; SVE:  LV: Setting widening decision to Scalarize for VF vscale x 2 and instruction: %val = load i32, ptr %src, align 4
; SVE:  LV: Memory widening: calculating best strategy for store i32 %val, ptr %arrayidx, align 4
; SVE:  LV: Memory widening: can be widened normally with cost 1
; SVE:  LV: Setting widening decision to Widen for VF vscale x 2 and instruction: store i32 %val, ptr %arrayidx, align 4
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %val = load i32, ptr %src, align 4
  %arrayidx = getelementptr inbounds i32, ptr %dst, i64 %iv
  store i32 %val, ptr %arrayidx, align 4
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %n
  br i1 %exitcond.not, label %for.end, label %for.body

for.end:
  ret void
}

define void @interleaved_load(ptr noalias readonly %src, ptr noalias %dst, i64 %n) {
; FIXED-LABEL: 'interleaved_load'
; FIXED:  LV: Memory widening: calculating best strategy for %load.0 = load i16, ptr %gep.0, align 2
; FIXED:  LV: Memory widening: InterleaveCost = 2, GatherScatterCost = Invalid, ScalarizationCost = 32
; FIXED:  LV: Setting widening decision to Interleave for VF 4 and instruction: %load.0 = load i16, ptr %gep.0, align 2
; FIXED:  LV: Setting widening decision to Interleave for VF 4 and instruction: %load.1 = load i16, ptr %gep.1, align 2
; FIXED-NOT:  LV: Memory widening: calculating best strategy for %load.1 = load i16, ptr %gep.1, align 2
; FIXED:  LV: Memory widening: calculating best strategy for store i16 %add, ptr %dst.gep, align 2
; FIXED:  LV: Memory widening: can be widened normally with cost 1
; FIXED:  LV: Setting widening decision to Widen for VF 4 and instruction: store i16 %add, ptr %dst.gep, align 2
;
; SVE-LABEL: 'interleaved_load'
; SVE:  LV: Memory widening: calculating best strategy for %load.0 = load i16, ptr %gep.0, align 2
; SVE:  LV: Memory widening: InterleaveCost = 5, GatherScatterCost = 42, ScalarizationCost = Invalid
; SVE:  LV: Setting widening decision to Interleave for VF vscale x 2 and instruction: %load.0 = load i16, ptr %gep.0, align 2
; SVE:  LV: Setting widening decision to Interleave for VF vscale x 2 and instruction: %load.1 = load i16, ptr %gep.1, align 2
; SVE-NOT:  LV: Memory widening: calculating best strategy for %load.1 = load i16, ptr %gep.1, align 2
; SVE:  LV: Memory widening: calculating best strategy for store i16 %add, ptr %dst.gep, align 2
; SVE:  LV: Memory widening: can be widened normally with cost 1
; SVE:  LV: Setting widening decision to Widen for VF vscale x 2 and instruction: store i16 %add, ptr %dst.gep, align 2
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv2 = shl nuw nsw i64 %iv, 1
  %gep.0 = getelementptr inbounds i16, ptr %src, i64 %iv2
  %load.0 = load i16, ptr %gep.0, align 2
  %idx.1 = or disjoint i64 %iv2, 1
  %gep.1 = getelementptr inbounds i16, ptr %src, i64 %idx.1
  %load.1 = load i16, ptr %gep.1, align 2
  %add = add i16 %load.0, %load.1
  %dst.gep = getelementptr inbounds i16, ptr %dst, i64 %iv
  store i16 %add, ptr %dst.gep, align 2
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %n
  br i1 %exitcond.not, label %for.end, label %for.body

for.end:
  ret void
}

define void @interleave_member_update() {
; FIXED-LABEL: 'interleave_member_update'
; FIXED:  LV: Memory widening: calculating best strategy for %idx = load i32, ptr %arrayidx, align 8
; FIXED:  LV: Memory widening: InterleaveCost = 2, GatherScatterCost = Invalid, ScalarizationCost = 16
; FIXED:  LV: Setting widening decision to Interleave for VF 4 and instruction: %idx = load i32, ptr %arrayidx, align 8
; FIXED:  LV: Memory widening: calculating best strategy for %val = load i32, ptr %gather.ptr, align 4
; FIXED:  LV: Memory widening: InterleaveCost = Invalid, GatherScatterCost = Invalid, ScalarizationCost = 52
; FIXED:  LV: Setting widening decision to Scalarize for VF 4 and instruction: %val = load i32, ptr %gather.ptr, align 4
; FIXED:  LV: Memory widening: calculating best strategy for %addr.idx = load i32, ptr %addr.idx.ptr, align 4
; FIXED:  LV: Memory widening: can be widened normally with cost 1
; FIXED:  LV: Setting widening decision to Widen for VF 4 and instruction: %addr.idx = load i32, ptr %addr.idx.ptr, align 4
; FIXED:  LV: Memory widening: calculating best strategy for store i32 %val, ptr %scatter.ptr, align 4
; FIXED:  LV: Memory widening: InterleaveCost = Invalid, GatherScatterCost = Invalid, ScalarizationCost = 52
; FIXED:  LV: Setting widening decision to Scalarize for VF 4 and instruction: store i32 %val, ptr %scatter.ptr, align 4
; FIXED:  LV: Memory widening: updating decision for load %addr.idx = load i32, ptr %addr.idx.ptr, align 4
; FIXED:  LV: Setting widening decision to Scalarize for VF 4 and instruction: %addr.idx = load i32, ptr %addr.idx.ptr, align 4
; FIXED:  LV: Memory widening: updating decision for interleave member %idx = load i32, ptr %arrayidx, align 8
; FIXED:  LV: Setting widening decision to Scalarize for VF 4 and instruction: %idx = load i32, ptr %arrayidx, align 8
;
; SVE-LABEL: 'interleave_member_update'
; SVE:  LV: Memory widening: calculating best strategy for %idx = load i32, ptr %arrayidx, align 8
; SVE:  LV: Memory widening: InterleaveCost = 5, GatherScatterCost = 21, ScalarizationCost = Invalid
; SVE:  LV: Setting widening decision to Interleave for VF vscale x 2 and instruction: %idx = load i32, ptr %arrayidx, align 8
; SVE:  LV: Memory widening: calculating best strategy for %val = load i32, ptr %gather.ptr, align 4
; SVE:  LV: Memory widening: InterleaveCost = Invalid, GatherScatterCost = 21, ScalarizationCost = Invalid
; SVE:  LV: Setting widening decision to GatherScatter for VF vscale x 2 and instruction: %val = load i32, ptr %gather.ptr, align 4
; SVE:  LV: Memory widening: calculating best strategy for %addr.idx = load i32, ptr %addr.idx.ptr, align 4
; SVE:  LV: Memory widening: can be widened normally with cost 1
; SVE:  LV: Setting widening decision to Widen for VF vscale x 2 and instruction: %addr.idx = load i32, ptr %addr.idx.ptr, align 4
; SVE:  LV: Memory widening: calculating best strategy for store i32 %val, ptr %scatter.ptr, align 4
; SVE:  LV: Memory widening: InterleaveCost = Invalid, GatherScatterCost = 21, ScalarizationCost = Invalid
; SVE:  LV: Setting widening decision to GatherScatter for VF vscale x 2 and instruction: store i32 %val, ptr %scatter.ptr, align 4
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %iv2 = shl nsw i64 %iv, 1
  %arrayidx = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 %iv2
  %idx = load i32, ptr %arrayidx, align 8
  %idxprom = sext i32 %idx to i64
  %gather.ptr = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 %idxprom
  %val = load i32, ptr %gather.ptr, align 4
  %addr.idx.ptr = getelementptr inbounds [2048 x i32], ptr @d, i64 0, i64 %iv
  %addr.idx = load i32, ptr %addr.idx.ptr, align 4
  %scatter.idx = sext i32 %addr.idx to i64
  %scatter.ptr = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 %scatter.idx
  store i32 %val, ptr %scatter.ptr, align 4
  %iv.next = add i64 %iv, 1
  %lftr.wideiv = trunc i64 %iv.next to i32
  %exitcond = icmp eq i32 %lftr.wideiv, 256
  br i1 %exitcond, label %for.end, label %for.body

for.end:
  ret void
}

define void @reverse_load_scatter(ptr noalias %src, ptr noalias %dst, i64 %n) {
; FIXED-LABEL: 'reverse_load_scatter'
; FIXED:  LV: Memory widening: calculating best strategy for %val = load i64, ptr %src.ptr, align 8
; FIXED:  LV: Memory widening: can be widened normally with cost 4
; FIXED:  LV: Setting widening decision to Widen_Reverse for VF 4 and instruction: %val = load i64, ptr %src.ptr, align 8
; FIXED:  LV: Memory widening: calculating best strategy for store i64 %val, ptr %dst.ptr, align 8
; FIXED:  LV: Memory widening: InterleaveCost = Invalid, GatherScatterCost = Invalid, ScalarizationCost = 60
; FIXED:  LV: Setting widening decision to Scalarize for VF 4 and instruction: store i64 %val, ptr %dst.ptr, align 8
; FIXED:  LV: Memory widening: updating decision for load %val = load i64, ptr %src.ptr, align 8
; FIXED:  LV: Setting widening decision to Scalarize for VF 4 and instruction: %val = load i64, ptr %src.ptr, align 8
; FIXED:  LV: Memory widening: updating decision for load user store i64 %val, ptr %dst.ptr, align 8
; FIXED:  LV: Setting widening decision to Scalarize for VF 4 and instruction: store i64 %val, ptr %dst.ptr, align 8
;
; SVE-LABEL: 'reverse_load_scatter'
; SVE:  LV: Memory widening: calculating best strategy for %val = load i64, ptr %src.ptr, align 8
; SVE:  LV: Memory widening: can be widened normally with cost 2
; SVE:  LV: Setting widening decision to Widen_Reverse for VF vscale x 2 and instruction: %val = load i64, ptr %src.ptr, align 8
; SVE:  LV: Memory widening: calculating best strategy for store i64 %val, ptr %dst.ptr, align 8
; SVE:  LV: Memory widening: InterleaveCost = Invalid, GatherScatterCost = 21, ScalarizationCost = Invalid
; SVE:  LV: Setting widening decision to GatherScatter for VF vscale x 2 and instruction: store i64 %val, ptr %dst.ptr, align 8
;
entry:
  br label %loop

loop:
  %iv.dec = phi i64 [ %n, %entry ], [ %iv.dec.next, %loop ]
  %src.ptr = getelementptr inbounds i64, ptr %src, i64 %iv.dec
  %val = load i64, ptr %src.ptr, align 8
  %dst.ptr = getelementptr inbounds i64, ptr %dst, i64 %val
  store i64 %val, ptr %dst.ptr, align 8
  %iv.dec.next = add i64 %iv.dec, -1
  %done = icmp eq i64 %iv.dec.next, 0
  br i1 %done, label %exit, label %loop

exit:
  ret void
}

define double @gather_load(ptr noalias readonly %data, ptr noalias readonly %offset, i64 %n) {
; FIXED-LABEL: 'gather_load'
; FIXED:  LV: Memory widening: calculating best strategy for %idx = load i32, ptr %idx.ptr, align 4
; FIXED:  LV: Memory widening: can be widened normally with cost 1
; FIXED:  LV: Setting widening decision to Widen for VF 4 and instruction: %idx = load i32, ptr %idx.ptr, align 4
; FIXED:  LV: Memory widening: calculating best strategy for %val = load double, ptr %gather.ptr, align 8
; FIXED:  LV: Memory widening: InterleaveCost = Invalid, GatherScatterCost = Invalid, ScalarizationCost = 48
; FIXED:  LV: Setting widening decision to Scalarize for VF 4 and instruction: %val = load double, ptr %gather.ptr, align 8
; FIXED:  LV: Memory widening: updating decision for load %idx = load i32, ptr %idx.ptr, align 4
; FIXED:  LV: Setting widening decision to Scalarize for VF 4 and instruction: %idx = load i32, ptr %idx.ptr, align 4
;
; SVE-LABEL: 'gather_load'
; SVE:  LV: Memory widening: calculating best strategy for %idx = load i32, ptr %idx.ptr, align 4
; SVE:  LV: Memory widening: can be widened normally with cost 1
; SVE:  LV: Setting widening decision to Widen for VF vscale x 2 and instruction: %idx = load i32, ptr %idx.ptr, align 4
; SVE:  LV: Memory widening: calculating best strategy for %val = load double, ptr %gather.ptr, align 8
; SVE:  LV: Memory widening: InterleaveCost = Invalid, GatherScatterCost = 21, ScalarizationCost = Invalid
; SVE:  LV: Setting widening decision to GatherScatter for VF vscale x 2 and instruction: %val = load double, ptr %gather.ptr, align 8
;
entry:
  br label %for.body

for.body:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
  %sum = phi double [ 0.000000e+00, %entry ], [ %add, %for.body ]
  %idx.ptr = getelementptr inbounds i32, ptr %offset, i64 %iv
  %idx = load i32, ptr %idx.ptr, align 4
  %idxprom = sext i32 %idx to i64
  %gather.ptr = getelementptr inbounds double, ptr %data, i64 %idxprom
  %val = load double, ptr %gather.ptr, align 8
  %add = fadd double %sum, %val
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, %n
  br i1 %exitcond.not, label %for.end, label %for.body

for.end:
  %res = phi double [ %add, %for.body ]
  ret double %res
}