llvm/test/Transforms/LoopVectorize/AArch64/force-scalable-vectorization-always-bf16.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64-linux-gnu -scalable-vectorization=always < %s | FileCheck %s

; Check that a scalable VF is chosen for bf16 (due to the `-scalable-vectorization=always` flag)
; whenever "+sve" is enabled in "target-features", regardless of whether "+bf16" is present.

define void @bfloat_loop(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
; CHECK-LABEL: define void @bfloat_loop(
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds bfloat, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x bfloat>, ptr [[TMP0]], align 4
; CHECK-NEXT:    [[TMP1:%.*]] = fadd <8 x bfloat> [[WIDE_LOAD]], splat (bfloat 1.000000e+00)
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT:    store <8 x bfloat> [[TMP1]], ptr [[TMP2]], align 4
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
; CHECK-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[FOR_END:.*]]
; CHECK:       [[FOR_END]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %i = phi i64 [ %inc, %for.body ], [ 0, %entry ]
  %b.i = getelementptr inbounds bfloat, ptr %b, i64 %i
  %0 = load bfloat, ptr %b.i, align 4
  %add = fadd bfloat %0, 1.0
  %a.i = getelementptr inbounds bfloat, ptr %a, i64 %i
  store bfloat %add, ptr %a.i, align 4
  %inc = add nuw nsw i64 %i, 1
  %exitcond.not = icmp eq i64 %inc, 4096
  br i1 %exitcond.not, label %for.end, label %for.body

for.end:
  ret void
}

define void @bfloat_loop_bf16(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) "target-features"="+bf16" {
; CHECK-LABEL: define void @bfloat_loop_bf16(
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds bfloat, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds bfloat, ptr [[TMP0]], i64 8
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x bfloat>, ptr [[TMP0]], align 4
; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x bfloat>, ptr [[TMP1]], align 4
; CHECK-NEXT:    [[TMP2:%.*]] = fadd <8 x bfloat> [[WIDE_LOAD]], splat (bfloat 1.000000e+00)
; CHECK-NEXT:    [[TMP3:%.*]] = fadd <8 x bfloat> [[WIDE_LOAD1]], splat (bfloat 1.000000e+00)
; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds bfloat, ptr [[TMP4]], i64 8
; CHECK-NEXT:    store <8 x bfloat> [[TMP2]], ptr [[TMP4]], align 4
; CHECK-NEXT:    store <8 x bfloat> [[TMP3]], ptr [[TMP5]], align 4
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[FOR_END:.*]]
; CHECK:       [[FOR_END]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %i = phi i64 [ %inc, %for.body ], [ 0, %entry ]
  %b.i = getelementptr inbounds bfloat, ptr %b, i64 %i
  %0 = load bfloat, ptr %b.i, align 4
  %add = fadd bfloat %0, 1.0
  %a.i = getelementptr inbounds bfloat, ptr %a, i64 %i
  store bfloat %add, ptr %a.i, align 4
  %inc = add nuw nsw i64 %i, 1
  %exitcond.not = icmp eq i64 %inc, 4096
  br i1 %exitcond.not, label %for.end, label %for.body

for.end:
  ret void
}

define void @bfloat_loop_sve(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) "target-features"="+sve" {
; CHECK-LABEL: define void @bfloat_loop_sve(
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP1]]
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP1]]
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds bfloat, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x bfloat>, ptr [[TMP3]], align 4
; CHECK-NEXT:    [[TMP4:%.*]] = fadd <vscale x 8 x bfloat> [[WIDE_LOAD]], splat (bfloat 1.000000e+00)
; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT:    store <vscale x 8 x bfloat> [[TMP4]], ptr [[TMP5]], align 4
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
; CHECK:       [[FOR_BODY]]:
; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT:    [[B_I:%.*]] = getelementptr inbounds bfloat, ptr [[B]], i64 [[I]]
; CHECK-NEXT:    [[TMP7:%.*]] = load bfloat, ptr [[B_I]], align 4
; CHECK-NEXT:    [[ADD:%.*]] = fadd bfloat [[TMP7]], 1.000000e+00
; CHECK-NEXT:    [[A_I:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[I]]
; CHECK-NEXT:    store bfloat [[ADD]], ptr [[A_I]], align 4
; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], 4096
; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK:       [[FOR_END]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %i = phi i64 [ %inc, %for.body ], [ 0, %entry ]
  %b.i = getelementptr inbounds bfloat, ptr %b, i64 %i
  %0 = load bfloat, ptr %b.i, align 4
  %add = fadd bfloat %0, 1.0
  %a.i = getelementptr inbounds bfloat, ptr %a, i64 %i
  store bfloat %add, ptr %a.i, align 4
  %inc = add nuw nsw i64 %i, 1
  %exitcond.not = icmp eq i64 %inc, 4096
  br i1 %exitcond.not, label %for.end, label %for.body

for.end:
  ret void
}

define void @bfloat_loop_sve_bf16(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) "target-features"="+sve,+bf16" {
; CHECK-LABEL: define void @bfloat_loop_sve_bf16(
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]]) #[[ATTR2:[0-9]+]] {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP1]]
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP1]]
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds bfloat, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds bfloat, ptr [[TMP4]], i64 [[TMP2]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x bfloat>, ptr [[TMP4]], align 4
; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x bfloat>, ptr [[TMP5]], align 4
; CHECK-NEXT:    [[TMP6:%.*]] = fadd <vscale x 8 x bfloat> [[WIDE_LOAD]], splat (bfloat 1.000000e+00)
; CHECK-NEXT:    [[TMP7:%.*]] = fadd <vscale x 8 x bfloat> [[WIDE_LOAD1]], splat (bfloat 1.000000e+00)
; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds bfloat, ptr [[TMP8]], i64 [[TMP2]]
; CHECK-NEXT:    store <vscale x 8 x bfloat> [[TMP6]], ptr [[TMP8]], align 4
; CHECK-NEXT:    store <vscale x 8 x bfloat> [[TMP7]], ptr [[TMP9]], align 4
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
; CHECK:       [[FOR_BODY]]:
; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT:    [[B_I:%.*]] = getelementptr inbounds bfloat, ptr [[B]], i64 [[I]]
; CHECK-NEXT:    [[TMP11:%.*]] = load bfloat, ptr [[B_I]], align 4
; CHECK-NEXT:    [[ADD:%.*]] = fadd bfloat [[TMP11]], 1.000000e+00
; CHECK-NEXT:    [[A_I:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[I]]
; CHECK-NEXT:    store bfloat [[ADD]], ptr [[A_I]], align 4
; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], 4096
; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK:       [[FOR_END]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %for.body

for.body:
  %i = phi i64 [ %inc, %for.body ], [ 0, %entry ]
  %b.i = getelementptr inbounds bfloat, ptr %b, i64 %i
  %0 = load bfloat, ptr %b.i, align 4
  %add = fadd bfloat %0, 1.0
  %a.i = getelementptr inbounds bfloat, ptr %a, i64 %i
  store bfloat %add, ptr %a.i, align 4
  %inc = add nuw nsw i64 %i, 1
  %exitcond.not = icmp eq i64 %inc, 4096
  br i1 %exitcond.not, label %for.end, label %for.body

for.end:
  ret void
}