test/CodeGen/AArch64/sve-ld1r.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
;
; Check that ldr1* instruction is generated to splat scalar during load,
; rather than mov from scalar to vector register (which would require the vector unit).
;
; one-off: ld1r_stack checks that ldr1b works with stack objects.
;
; Test axes:
;   types = [i8, i16, i32, i64, half, float, double]
;   methods = [direct load, gep upper bound - 1, gep out of range x {neg,pos}, sext..., zext..., unpacked_floats...]
;

@g8 = external global i8

; One-off test for splatted value coming from stack load.
define <vscale x 16 x i8> @ld1r_stack() {
; CHECK-LABEL: ld1r_stack:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #16
; CHECK-NEXT:    .cfi_def_cfa_offset 16
; CHECK-NEXT:    adrp x8, :got:g8
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    ldr x8, [x8, :got_lo12:g8]
; CHECK-NEXT:    ldrb w8, [x8]
; CHECK-NEXT:    strb w8, [sp, #12]
; CHECK-NEXT:    ld1rb { z0.b }, p0/z, [sp, #14]
; CHECK-NEXT:    add sp, sp, #16
; CHECK-NEXT:    ret
  %valp = alloca i8
  %valp2  = load volatile i8, i8* @g8
  store volatile i8 %valp2, i8* %valp
  %valp3 = getelementptr i8, i8* %valp, i32 2
  %val = load i8, i8* %valp3
  %1 = insertelement <vscale x 16 x i8> undef, i8 %val, i32 0
  %2 = shufflevector <vscale x 16 x i8> %1, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
  ret <vscale x 16 x i8> %2
}

define <vscale x 16 x i8> @ld1rb(i8* %valp) {
; CHECK-LABEL: ld1rb:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    ld1rb { z0.b }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i8, i8* %valp
  %ins = insertelement <vscale x 16 x i8> undef, i8 %val, i32 0
  %shf = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
  ret <vscale x 16 x i8> %shf
}

define <vscale x 16 x i8> @ld1rb_gep(i8* %valp) {
; CHECK-LABEL: ld1rb_gep:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    ld1rb { z0.b }, p0/z, [x0, #63]
; CHECK-NEXT:    ret
  %valp2 = getelementptr i8, i8* %valp, i32 63
  %val = load i8, i8* %valp2
  %ins = insertelement <vscale x 16 x i8> undef, i8 %val, i32 0
  %shf = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
  ret <vscale x 16 x i8> %shf
}

define <vscale x 16 x i8> @ld1rb_gep_out_of_range_up(i8* %valp) {
; CHECK-LABEL: ld1rb_gep_out_of_range_up:
; CHECK:       // %bb.0:
; CHECK-NEXT:    add x8, x0, #64
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    ld1rb { z0.b }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr i8, i8* %valp, i32 64
  %val = load i8, i8* %valp2
  %ins = insertelement <vscale x 16 x i8> undef, i8 %val, i32 0
  %shf = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
  ret <vscale x 16 x i8> %shf
}

define <vscale x 16 x i8> @ld1rb_gep_out_of_range_down(i8* %valp) {
; CHECK-LABEL: ld1rb_gep_out_of_range_down:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub x8, x0, #1
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    ld1rb { z0.b }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr i8, i8* %valp, i32 -1
  %val = load i8, i8* %valp2
  %ins = insertelement <vscale x 16 x i8> undef, i8 %val, i32 0
  %shf = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
  ret <vscale x 16 x i8> %shf
}

define <vscale x 8 x i16> @ld1rb_i8_i16_zext(i8* %valp) {
; CHECK-LABEL: ld1rb_i8_i16_zext:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    ld1rb { z0.h }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i8, i8* %valp
  %ext = zext i8 %val to i16
  %ins = insertelement <vscale x 8 x i16> undef, i16 %ext, i32 0
  %shf = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
  ret <vscale x 8 x i16> %shf
}

define <vscale x 8 x i16> @ld1rb_i8_i16_sext(i8* %valp) {
; CHECK-LABEL: ld1rb_i8_i16_sext:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    ld1rsb { z0.h }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i8, i8* %valp
  %ext = sext i8 %val to i16
  %ins = insertelement <vscale x 8 x i16> undef, i16 %ext, i32 0
  %shf = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
  ret <vscale x 8 x i16> %shf
}

define <vscale x 4 x i32> @ld1rb_i8_i32_zext(i8* %valp) {
; CHECK-LABEL: ld1rb_i8_i32_zext:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rb { z0.s }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i8, i8* %valp
  %ext = zext i8 %val to i32
  %ins = insertelement <vscale x 4 x i32> undef, i32 %ext, i32 0
  %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x i32> %shf
}

define <vscale x 4 x i32> @ld1rb_i8_i32_sext(i8* %valp) {
; CHECK-LABEL: ld1rb_i8_i32_sext:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rsb { z0.s }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i8, i8* %valp
  %ext = sext i8 %val to i32
  %ins = insertelement <vscale x 4 x i32> undef, i32 %ext, i32 0
  %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x i32> %shf
}

define <vscale x 2 x i64> @ld1rb_i8_i64_zext(i8* %valp) {
; CHECK-LABEL: ld1rb_i8_i64_zext:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rb { z0.d }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i8, i8* %valp
  %ext = zext i8 %val to i64
  %ins = insertelement <vscale x 2 x i64> undef, i64 %ext, i32 0
  %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x i64> %shf
}

define <vscale x 2 x i64> @ld1rb_i8_i64_sext(i8* %valp) {
; CHECK-LABEL: ld1rb_i8_i64_sext:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rsb { z0.d }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i8, i8* %valp
  %ext = sext i8 %val to i64
  %ins = insertelement <vscale x 2 x i64> undef, i64 %ext, i32 0
  %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x i64> %shf
}

define <vscale x 8 x i16> @ld1rh(i16* %valp) {
; CHECK-LABEL: ld1rh:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    ld1rh { z0.h }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i16, i16* %valp
  %ins = insertelement <vscale x 8 x i16> undef, i16 %val, i32 0
  %shf = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
  ret <vscale x 8 x i16> %shf
}

define <vscale x 8 x i16> @ld1rh_gep(i16* %valp) {
; CHECK-LABEL: ld1rh_gep:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    ld1rh { z0.h }, p0/z, [x0, #126]
; CHECK-NEXT:    ret
  %valp2 = getelementptr i16, i16* %valp, i32 63
  %val = load i16, i16* %valp2
  %ins = insertelement <vscale x 8 x i16> undef, i16 %val, i32 0
  %shf = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
  ret <vscale x 8 x i16> %shf
}

define <vscale x 8 x i16> @ld1rh_gep_out_of_range_up(i16* %valp) {
; CHECK-LABEL: ld1rh_gep_out_of_range_up:
; CHECK:       // %bb.0:
; CHECK-NEXT:    add x8, x0, #128
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    ld1rh { z0.h }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr i16, i16* %valp, i32 64
  %val = load i16, i16* %valp2
  %ins = insertelement <vscale x 8 x i16> undef, i16 %val, i32 0
  %shf = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
  ret <vscale x 8 x i16> %shf
}

define <vscale x 8 x i16> @ld1rh_gep_out_of_range_down(i16* %valp) {
; CHECK-LABEL: ld1rh_gep_out_of_range_down:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub x8, x0, #2
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    ld1rh { z0.h }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr i16, i16* %valp, i32 -1
  %val = load i16, i16* %valp2
  %ins = insertelement <vscale x 8 x i16> undef, i16 %val, i32 0
  %shf = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
  ret <vscale x 8 x i16> %shf
}

define <vscale x 4 x i32> @ld1rh_i16_i32_zext(i16* %valp) {
; CHECK-LABEL: ld1rh_i16_i32_zext:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rh { z0.s }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i16, i16* %valp
  %ext = zext i16 %val to i32
  %ins = insertelement <vscale x 4 x i32> undef, i32 %ext, i32 0
  %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x i32> %shf
}

define <vscale x 4 x i32> @ld1rh_i16_i32_sext(i16* %valp) {
; CHECK-LABEL: ld1rh_i16_i32_sext:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rsh { z0.s }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i16, i16* %valp
  %ext = sext i16 %val to i32
  %ins = insertelement <vscale x 4 x i32> undef, i32 %ext, i32 0
  %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x i32> %shf
}

define <vscale x 2 x i64> @ld1rh_i16_i64_zext(i16* %valp) {
; CHECK-LABEL: ld1rh_i16_i64_zext:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rh { z0.d }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i16, i16* %valp
  %ext = zext i16 %val to i64
  %ins = insertelement <vscale x 2 x i64> undef, i64 %ext, i32 0
  %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x i64> %shf
}

define <vscale x 2 x i64> @ld1rh_i16_i64_sext(i16* %valp) {
; CHECK-LABEL: ld1rh_i16_i64_sext:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rsh { z0.d }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i16, i16* %valp
  %ext = sext i16 %val to i64
  %ins = insertelement <vscale x 2 x i64> undef, i64 %ext, i32 0
  %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x i64> %shf
}

define <vscale x 4 x i32> @ld1rw(i32* %valp) {
; CHECK-LABEL: ld1rw:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rw { z0.s }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i32, i32* %valp
  %ins = insertelement <vscale x 4 x i32> undef, i32 %val, i32 0
  %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x i32> %shf
}

define <vscale x 4 x i32> @ld1rw_gep(i32* %valp) {
; CHECK-LABEL: ld1rw_gep:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rw { z0.s }, p0/z, [x0, #252]
; CHECK-NEXT:    ret
  %valp2 = getelementptr i32, i32* %valp, i32 63
  %val = load i32, i32* %valp2
  %ins = insertelement <vscale x 4 x i32> undef, i32 %val, i32 0
  %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x i32> %shf
}

define <vscale x 4 x i32> @ld1rw_gep_out_of_range_up(i32* %valp) {
; CHECK-LABEL: ld1rw_gep_out_of_range_up:
; CHECK:       // %bb.0:
; CHECK-NEXT:    add x8, x0, #256
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rw { z0.s }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr i32, i32* %valp, i32 64
  %val = load i32, i32* %valp2
  %ins = insertelement <vscale x 4 x i32> undef, i32 %val, i32 0
  %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x i32> %shf
}

define <vscale x 4 x i32> @ld1rw_gep_out_of_range_down(i32* %valp) {
; CHECK-LABEL: ld1rw_gep_out_of_range_down:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub x8, x0, #4
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rw { z0.s }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr i32, i32* %valp, i32 -1
  %val = load i32, i32* %valp2
  %ins = insertelement <vscale x 4 x i32> undef, i32 %val, i32 0
  %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x i32> %shf
}

define <vscale x 2 x i64> @ld1rw_i32_i64_zext(i32* %valp) {
; CHECK-LABEL: ld1rw_i32_i64_zext:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rw { z0.d }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i32, i32* %valp
  %ext = zext i32 %val to i64
  %ins = insertelement <vscale x 2 x i64> undef, i64 %ext, i32 0
  %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x i64> %shf
}

define <vscale x 2 x i64> @ld1rw_i32_i64_sext(i32* %valp) {
; CHECK-LABEL: ld1rw_i32_i64_sext:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rsw { z0.d }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i32, i32* %valp
  %ext = sext i32 %val to i64
  %ins = insertelement <vscale x 2 x i64> undef, i64 %ext, i32 0
  %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x i64> %shf
}

define <vscale x 2 x i64> @ld1rd(i64* %valp) {
; CHECK-LABEL: ld1rd:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rd { z0.d }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load i64, i64* %valp
  %ins = insertelement <vscale x 2 x i64> undef, i64 %val, i32 0
  %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x i64> %shf
}

define <vscale x 2 x i64> @ld1rd_gep(i64* %valp) {
; CHECK-LABEL: ld1rd_gep:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rd { z0.d }, p0/z, [x0, #504]
; CHECK-NEXT:    ret
  %valp2 = getelementptr i64, i64* %valp, i32 63
  %val = load i64, i64* %valp2
  %ins = insertelement <vscale x 2 x i64> undef, i64 %val, i32 0
  %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x i64> %shf
}

define <vscale x 2 x i64> @ld1rd_gep_out_of_range_up(i64* %valp) {
; CHECK-LABEL: ld1rd_gep_out_of_range_up:
; CHECK:       // %bb.0:
; CHECK-NEXT:    add x8, x0, #512
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rd { z0.d }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr i64, i64* %valp, i32 64
  %val = load i64, i64* %valp2
  %ins = insertelement <vscale x 2 x i64> undef, i64 %val, i32 0
  %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x i64> %shf
}

define <vscale x 2 x i64> @ld1rd_gep_out_of_range_down(i64* %valp) {
; CHECK-LABEL: ld1rd_gep_out_of_range_down:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub x8, x0, #8
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rd { z0.d }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr i64, i64* %valp, i32 -1
  %val = load i64, i64* %valp2
  %ins = insertelement <vscale x 2 x i64> undef, i64 %val, i32 0
  %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x i64> %shf
}

define <vscale x 8 x half> @ld1rh_half(half* %valp) {
; CHECK-LABEL: ld1rh_half:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    ld1rh { z0.h }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load half, half* %valp
  %ins = insertelement <vscale x 8 x half> undef, half %val, i32 0
  %shf = shufflevector <vscale x 8 x half> %ins, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
  ret <vscale x 8 x half> %shf
}

define <vscale x 8 x half> @ld1rh_half_gep(half* %valp) {
; CHECK-LABEL: ld1rh_half_gep:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    ld1rh { z0.h }, p0/z, [x0, #126]
; CHECK-NEXT:    ret
  %valp2 = getelementptr half, half* %valp, i32 63
  %val = load half, half* %valp2
  %ins = insertelement <vscale x 8 x half> undef, half %val, i32 0
  %shf = shufflevector <vscale x 8 x half> %ins, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
  ret <vscale x 8 x half> %shf
}

define <vscale x 8 x half> @ld1rh_half_gep_out_of_range_up(half* %valp) {
; CHECK-LABEL: ld1rh_half_gep_out_of_range_up:
; CHECK:       // %bb.0:
; CHECK-NEXT:    add x8, x0, #128
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    ld1rh { z0.h }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr half, half* %valp, i32 64
  %val = load half, half* %valp2
  %ins = insertelement <vscale x 8 x half> undef, half %val, i32 0
  %shf = shufflevector <vscale x 8 x half> %ins, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
  ret <vscale x 8 x half> %shf
}

define <vscale x 8 x half> @ld1rh_half_gep_out_of_range_down(half* %valp) {
; CHECK-LABEL: ld1rh_half_gep_out_of_range_down:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub x8, x0, #2
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    ld1rh { z0.h }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr half, half* %valp, i32 -1
  %val = load half, half* %valp2
  %ins = insertelement <vscale x 8 x half> undef, half %val, i32 0
  %shf = shufflevector <vscale x 8 x half> %ins, <vscale x 8 x half> undef, <vscale x 8 x i32> zeroinitializer
  ret <vscale x 8 x half> %shf
}

define <vscale x 4 x half> @ld1rh_half_unpacked4(half* %valp) {
; CHECK-LABEL: ld1rh_half_unpacked4:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rh { z0.s }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load half, half* %valp
  %ins = insertelement <vscale x 4 x half> undef, half %val, i32 0
  %shf = shufflevector <vscale x 4 x half> %ins, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x half> %shf
}

define <vscale x 4 x half> @ld1rh_half_unpacked4_gep(half* %valp) {
; CHECK-LABEL: ld1rh_half_unpacked4_gep:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rh { z0.s }, p0/z, [x0, #126]
; CHECK-NEXT:    ret
  %valp2 = getelementptr half, half* %valp, i32 63
  %val = load half, half* %valp2
  %ins = insertelement <vscale x 4 x half> undef, half %val, i32 0
  %shf = shufflevector <vscale x 4 x half> %ins, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x half> %shf
}

define <vscale x 4 x half> @ld1rh_half_unpacked4_gep_out_of_range_up(half* %valp) {
; CHECK-LABEL: ld1rh_half_unpacked4_gep_out_of_range_up:
; CHECK:       // %bb.0:
; CHECK-NEXT:    add x8, x0, #128
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rh { z0.s }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr half, half* %valp, i32 64
  %val = load half, half* %valp2
  %ins = insertelement <vscale x 4 x half> undef, half %val, i32 0
  %shf = shufflevector <vscale x 4 x half> %ins, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x half> %shf
}

define <vscale x 4 x half> @ld1rh_half_unpacked4_gep_out_of_range_down(half* %valp) {
; CHECK-LABEL: ld1rh_half_unpacked4_gep_out_of_range_down:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub x8, x0, #2
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rh { z0.s }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr half, half* %valp, i32 -1
  %val = load half, half* %valp2
  %ins = insertelement <vscale x 4 x half> undef, half %val, i32 0
  %shf = shufflevector <vscale x 4 x half> %ins, <vscale x 4 x half> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x half> %shf
}

define <vscale x 2 x half> @ld1rh_half_unpacked2(half* %valp) {
; CHECK-LABEL: ld1rh_half_unpacked2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rh { z0.d }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load half, half* %valp
  %ins = insertelement <vscale x 2 x half> undef, half %val, i32 0
  %shf = shufflevector <vscale x 2 x half> %ins, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x half> %shf
}

define <vscale x 2 x half> @ld1rh_half_unpacked2_gep(half* %valp) {
; CHECK-LABEL: ld1rh_half_unpacked2_gep:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rh { z0.d }, p0/z, [x0, #126]
; CHECK-NEXT:    ret
  %valp2 = getelementptr half, half* %valp, i32 63
  %val = load half, half* %valp2
  %ins = insertelement <vscale x 2 x half> undef, half %val, i32 0
  %shf = shufflevector <vscale x 2 x half> %ins, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x half> %shf
}

define <vscale x 2 x half> @ld1rh_half_unpacked2_gep_out_of_range_up(half* %valp) {
; CHECK-LABEL: ld1rh_half_unpacked2_gep_out_of_range_up:
; CHECK:       // %bb.0:
; CHECK-NEXT:    add x8, x0, #128
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rh { z0.d }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr half, half* %valp, i32 64
  %val = load half, half* %valp2
  %ins = insertelement <vscale x 2 x half> undef, half %val, i32 0
  %shf = shufflevector <vscale x 2 x half> %ins, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x half> %shf
}

define <vscale x 2 x half> @ld1rh_half_unpacked2_gep_out_of_range_down(half* %valp) {
; CHECK-LABEL: ld1rh_half_unpacked2_gep_out_of_range_down:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub x8, x0, #2
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rh { z0.d }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr half, half* %valp, i32 -1
  %val = load half, half* %valp2
  %ins = insertelement <vscale x 2 x half> undef, half %val, i32 0
  %shf = shufflevector <vscale x 2 x half> %ins, <vscale x 2 x half> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x half> %shf
}

define <vscale x 4 x float> @ld1rw_float(float* %valp) {
; CHECK-LABEL: ld1rw_float:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rw { z0.s }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load float, float* %valp
  %ins = insertelement <vscale x 4 x float> undef, float %val, i32 0
  %shf = shufflevector <vscale x 4 x float> %ins, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x float> %shf
}

define <vscale x 4 x float> @ld1rw_float_gep(float* %valp) {
; CHECK-LABEL: ld1rw_float_gep:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rw { z0.s }, p0/z, [x0, #252]
; CHECK-NEXT:    ret
  %valp2 = getelementptr float, float* %valp, i32 63
  %val = load float, float* %valp2
  %ins = insertelement <vscale x 4 x float> undef, float %val, i32 0
  %shf = shufflevector <vscale x 4 x float> %ins, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x float> %shf
}

define <vscale x 4 x float> @ld1rw_float_gep_out_of_range_up(float* %valp) {
; CHECK-LABEL: ld1rw_float_gep_out_of_range_up:
; CHECK:       // %bb.0:
; CHECK-NEXT:    add x8, x0, #256
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rw { z0.s }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr float, float* %valp, i32 64
  %val = load float, float* %valp2
  %ins = insertelement <vscale x 4 x float> undef, float %val, i32 0
  %shf = shufflevector <vscale x 4 x float> %ins, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x float> %shf
}

define <vscale x 4 x float> @ld1rw_float_gep_out_of_range_down(float* %valp) {
; CHECK-LABEL: ld1rw_float_gep_out_of_range_down:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub x8, x0, #4
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rw { z0.s }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr float, float* %valp, i32 -1
  %val = load float, float* %valp2
  %ins = insertelement <vscale x 4 x float> undef, float %val, i32 0
  %shf = shufflevector <vscale x 4 x float> %ins, <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
  ret <vscale x 4 x float> %shf
}

define <vscale x 2 x float> @ld1rw_float_unpacked2(float* %valp) {
; CHECK-LABEL: ld1rw_float_unpacked2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rw { z0.d }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load float, float* %valp
  %ins = insertelement <vscale x 2 x float> undef, float %val, i32 0
  %shf = shufflevector <vscale x 2 x float> %ins, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x float> %shf
}

define <vscale x 2 x float> @ld1rw_float_unpacked2_gep(float* %valp) {
; CHECK-LABEL: ld1rw_float_unpacked2_gep:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rw { z0.d }, p0/z, [x0, #252]
; CHECK-NEXT:    ret
  %valp2 = getelementptr float, float* %valp, i32 63
  %val = load float, float* %valp2
  %ins = insertelement <vscale x 2 x float> undef, float %val, i32 0
  %shf = shufflevector <vscale x 2 x float> %ins, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x float> %shf
}

define <vscale x 2 x float> @ld1rw_float_unpacked2_gep_out_of_range_up(float* %valp) {
; CHECK-LABEL: ld1rw_float_unpacked2_gep_out_of_range_up:
; CHECK:       // %bb.0:
; CHECK-NEXT:    add x8, x0, #256
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rw { z0.d }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr float, float* %valp, i32 64
  %val = load float, float* %valp2
  %ins = insertelement <vscale x 2 x float> undef, float %val, i32 0
  %shf = shufflevector <vscale x 2 x float> %ins, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x float> %shf
}

define <vscale x 2 x float> @ld1rw_float_unpacked2_gep_out_of_range_down(float* %valp) {
; CHECK-LABEL: ld1rw_float_unpacked2_gep_out_of_range_down:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub x8, x0, #4
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rw { z0.d }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr float, float* %valp, i32 -1
  %val = load float, float* %valp2
  %ins = insertelement <vscale x 2 x float> undef, float %val, i32 0
  %shf = shufflevector <vscale x 2 x float> %ins, <vscale x 2 x float> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x float> %shf
}

define <vscale x 2 x double> @ld1rd_double(double* %valp) {
; CHECK-LABEL: ld1rd_double:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rd { z0.d }, p0/z, [x0]
; CHECK-NEXT:    ret
  %val = load double, double* %valp
  %ins = insertelement <vscale x 2 x double> undef, double %val, i32 0
  %shf = shufflevector <vscale x 2 x double> %ins, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x double> %shf
}

define <vscale x 2 x double> @ld1rd_double_gep(double* %valp) {
; CHECK-LABEL: ld1rd_double_gep:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rd { z0.d }, p0/z, [x0, #504]
; CHECK-NEXT:    ret
  %valp2 = getelementptr double, double* %valp, i32 63
  %val = load double, double* %valp2
  %ins = insertelement <vscale x 2 x double> undef, double %val, i32 0
  %shf = shufflevector <vscale x 2 x double> %ins, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x double> %shf
}

define <vscale x 2 x double> @ld1rd_double_gep_out_of_range_up(double* %valp) {
; CHECK-LABEL: ld1rd_double_gep_out_of_range_up:
; CHECK:       // %bb.0:
; CHECK-NEXT:    add x8, x0, #512
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rd { z0.d }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr double, double* %valp, i32 64
  %val = load double, double* %valp2
  %ins = insertelement <vscale x 2 x double> undef, double %val, i32 0
  %shf = shufflevector <vscale x 2 x double> %ins, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x double> %shf
}

define <vscale x 2 x double> @ld1rd_double_gep_out_of_range_down(double* %valp) {
; CHECK-LABEL: ld1rd_double_gep_out_of_range_down:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub x8, x0, #8
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rd { z0.d }, p0/z, [x8]
; CHECK-NEXT:    ret
  %valp2 = getelementptr double, double* %valp, i32 -1
  %val = load double, double* %valp2
  %ins = insertelement <vscale x 2 x double> undef, double %val, i32 0
  %shf = shufflevector <vscale x 2 x double> %ins, <vscale x 2 x double> undef, <vscale x 2 x i32> zeroinitializer
  ret <vscale x 2 x double> %shf
}

define <vscale x 2 x double> @dupq_ld1rqd_f64(<2 x double>* %a) {
; CHECK-LABEL: dupq_ld1rqd_f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rqd { z0.d }, p0/z, [x0]
; CHECK-NEXT:    ret
  %1 = load <2 x double>, <2 x double>* %a
  %2 = tail call fast <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> undef, <2 x double> %1, i64 0)
  %3 = tail call fast <vscale x 2 x double> @llvm.aarch64.sve.dupq.lane.nxv2f64(<vscale x 2 x double> %2, i64 0)
  ret <vscale x 2 x double> %3
}

define <vscale x 4 x float> @dupq_ld1rqw_f32(<4 x float>* %a) {
; CHECK-LABEL: dupq_ld1rqw_f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rqw { z0.s }, p0/z, [x0]
; CHECK-NEXT:    ret
  %1 = load <4 x float>, <4 x float>* %a
  %2 = tail call fast <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> %1, i64 0)
  %3 = tail call fast <vscale x 4 x float> @llvm.aarch64.sve.dupq.lane.nxv4f32(<vscale x 4 x float> %2, i64 0)
  ret <vscale x 4 x float> %3
}

define <vscale x 8 x half> @dupq_ld1rqh_f16(<8 x half>* %a) {
; CHECK-LABEL: dupq_ld1rqh_f16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    ld1rqh { z0.h }, p0/z, [x0]
; CHECK-NEXT:    ret
  %1 = load <8 x half>, <8 x half>* %a
  %2 = tail call fast <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v8f16(<vscale x 8 x half> undef, <8 x half> %1, i64 0)
  %3 = tail call fast <vscale x 8 x half> @llvm.aarch64.sve.dupq.lane.nxv8f16(<vscale x 8 x half> %2, i64 0)
  ret <vscale x 8 x half> %3
}

define <vscale x 8 x bfloat> @dupq_ld1rqh_bf16(<8 x bfloat>* %a) #0 {
; CHECK-LABEL: dupq_ld1rqh_bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    ld1rqh { z0.h }, p0/z, [x0]
; CHECK-NEXT:    ret
  %1 = load <8 x bfloat>, <8 x bfloat>* %a
  %2 = tail call fast <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat> undef, <8 x bfloat> %1, i64 0)
  %3 = tail call fast <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat> %2, i64 0)
  ret <vscale x 8 x bfloat> %3
}

define <vscale x 2 x i64> @dupq_ld1rqd_i64(<2 x i64>* %a) #0 {
; CHECK-LABEL: dupq_ld1rqd_i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    ld1rqd { z0.d }, p0/z, [x0]
; CHECK-NEXT:    ret
  %1 = load <2 x i64>, <2 x i64>* %a
  %2 = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> %1, i64 0)
  %3 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.dupq.lane.nxv2i64(<vscale x 2 x i64> %2, i64 0)
  ret <vscale x 2 x i64> %3
}

define <vscale x 4 x i32> @dupq_ld1rqw_i32(<4 x i32>* %a) #0 {
; CHECK-LABEL: dupq_ld1rqw_i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    ld1rqw { z0.s }, p0/z, [x0]
; CHECK-NEXT:    ret
  %1 = load <4 x i32>, <4 x i32>* %a
  %2 = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> undef, <4 x i32> %1, i64 0)
  %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.dupq.lane.nxv4i32(<vscale x 4 x i32> %2, i64 0)
  ret <vscale x 4 x i32> %3
}

define <vscale x 8 x i16> @dupq_ld1rqw_i16(<8 x i16>* %a) #0 {
; CHECK-LABEL: dupq_ld1rqw_i16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    ld1rqh { z0.h }, p0/z, [x0]
; CHECK-NEXT:    ret
  %1 = load <8 x i16>, <8 x i16>* %a
  %2 = tail call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v8i16(<vscale x 8 x i16> undef, <8 x i16> %1, i64 0)
  %3 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.dupq.lane.nxv8i16(<vscale x 8 x i16> %2, i64 0)
  ret <vscale x 8 x i16> %3
}

define <vscale x 16 x i8> @dupq_ld1rqw_i8(<16 x i8>* %a) #0 {
; CHECK-LABEL: dupq_ld1rqw_i8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    ld1rqb { z0.b }, p0/z, [x0]
; CHECK-NEXT:    ret
  %1 = load <16 x i8>, <16 x i8>* %a
  %2 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> %1, i64 0)
  %3 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.dupq.lane.nxv16i8(<vscale x 16 x i8> %2, i64 0)
  ret <vscale x 16 x i8> %3
}

declare <vscale x 16 x i8> @llvm.aarch64.sve.dupq.lane.nxv16i8(<vscale x 16 x i8>, i64)
declare <vscale x 8 x i16> @llvm.aarch64.sve.dupq.lane.nxv8i16(<vscale x 8 x i16>, i64)
declare <vscale x 4 x i32> @llvm.aarch64.sve.dupq.lane.nxv4i32(<vscale x 4 x i32>, i64)
declare <vscale x 2 x i64> @llvm.aarch64.sve.dupq.lane.nxv2i64(<vscale x 2 x i64>, i64)
declare <vscale x 8 x half> @llvm.aarch64.sve.dupq.lane.nxv8f16(<vscale x 8 x half>, i64)
declare <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat>, i64)
declare <vscale x 4 x float> @llvm.aarch64.sve.dupq.lane.nxv4f32(<vscale x 4 x float>, i64)
declare <vscale x 2 x double> @llvm.aarch64.sve.dupq.lane.nxv2f64(<vscale x 2 x double>, i64)

declare <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double>, <2 x double>, i64)
declare <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float>, <4 x float>, i64)
declare <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v8f16(<vscale x 8 x half>, <8 x half>, i64)
declare <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64>, <2 x i64>, i64)
declare <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32>, <4 x i32>, i64)
declare <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v8i16(<vscale x 8 x i16>, <8 x i16>, i64)
declare <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8>, <16 x i8>, i64)
declare <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat>, <8 x bfloat>, i64)

attributes #0 = { "target-features"="+sve,+bf16" }