; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16,CHECK-NOFP16-SD
; RUN: llc -mtriple=aarch64 -mattr=+fullfp16,+bf16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-FP16,CHECK-FP16-SD
; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16,CHECK-NOFP16-GI
; RUN: llc -mtriple=aarch64 -mattr=+fullfp16,+bf16 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-FP16,CHECK-FP16-GI

; CHECK-NOFP16-GI:       warning: Instruction selection used fallback path for stofp_i64_bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_i64_bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_i32_bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_i32_bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_i16_bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_i16_bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_i8_bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_i8_bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v2i64_v2bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v2i64_v2bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v3i64_v3bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v3i64_v3bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v4i64_v4bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v4i64_v4bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v8i64_v8bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v8i64_v8bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v16i64_v16bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v16i64_v16bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v32i64_v32bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v32i64_v32bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v2i32_v2bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v2i32_v2bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v3i32_v3bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v3i32_v3bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v4i32_v4bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v4i32_v4bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v8i32_v8bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v8i32_v8bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v16i32_v16bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v16i32_v16bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v32i32_v32bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v32i32_v32bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v2i16_v2bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v2i16_v2bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v3i16_v3bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v3i16_v3bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v4i16_v4bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v4i16_v4bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v8i16_v8bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v8i16_v8bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v16i16_v16bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v16i16_v16bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v32i16_v32bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v32i16_v32bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v2i8_v2bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v2i8_v2bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v3i8_v3bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v3i8_v3bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v4i8_v4bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v4i8_v4bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v8i8_v8bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v8i8_v8bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v16i8_v16bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v16i8_v16bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v32i8_v32bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v32i8_v32bf16
; CHECK-NOFP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v4i48_v4f16
;
; CHECK-FP16-GI:       warning: Instruction selection used fallback path for stofp_i64_bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_i64_bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_i32_bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_i32_bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_i16_bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_i16_bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_i8_bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_i8_bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v2i64_v2bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v2i64_v2bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v3i64_v3bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v3i64_v3bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v4i64_v4bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v4i64_v4bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v8i64_v8bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v8i64_v8bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v16i64_v16bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v16i64_v16bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v32i64_v32bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v32i64_v32bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v2i32_v2bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v2i32_v2bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v3i32_v3bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v3i32_v3bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v4i32_v4bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v4i32_v4bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v8i32_v8bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v8i32_v8bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v16i32_v16bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v16i32_v16bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v32i32_v32bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v32i32_v32bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v2i16_v2bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v2i16_v2bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v3i16_v3bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v3i16_v3bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v4i16_v4bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v4i16_v4bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v8i16_v8bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v8i16_v8bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v16i16_v16bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v16i16_v16bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v32i16_v32bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v32i16_v32bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v2i8_v2bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v2i8_v2bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v3i8_v3bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v3i8_v3bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v4i8_v4bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v4i8_v4bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v8i8_v8bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v8i8_v8bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v16i8_v16bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v16i8_v16bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v32i8_v32bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v32i8_v32bf16
; CHECK-FP16-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v4i48_v4f16

define bfloat @stofp_i64_bf16(i64 %a) {
; CHECK-NOFP16-LABEL: stofp_i64_bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    cmp x0, #0
; CHECK-NOFP16-NEXT:    and x11, x0, #0x8000000000000000
; CHECK-NOFP16-NEXT:    mov w8, #32767 // =0x7fff
; CHECK-NOFP16-NEXT:    cneg x9, x0, mi
; CHECK-NOFP16-NEXT:    lsr x10, x9, #53
; CHECK-NOFP16-NEXT:    cmp x10, #0
; CHECK-NOFP16-NEXT:    and x10, x9, #0xfffffffffffff000
; CHECK-NOFP16-NEXT:    csel x10, x10, x9, ne
; CHECK-NOFP16-NEXT:    scvtf d0, x10
; CHECK-NOFP16-NEXT:    cset w10, ne
; CHECK-NOFP16-NEXT:    tst x9, #0xfff
; CHECK-NOFP16-NEXT:    csel w10, wzr, w10, eq
; CHECK-NOFP16-NEXT:    fmov x9, d0
; CHECK-NOFP16-NEXT:    orr x9, x9, x11
; CHECK-NOFP16-NEXT:    orr x9, x9, x10
; CHECK-NOFP16-NEXT:    fmov d0, x9
; CHECK-NOFP16-NEXT:    fcvtxn s0, d0
; CHECK-NOFP16-NEXT:    fmov w9, s0
; CHECK-NOFP16-NEXT:    ubfx w10, w9, #16, #1
; CHECK-NOFP16-NEXT:    add w8, w9, w8
; CHECK-NOFP16-NEXT:    add w8, w10, w8
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    fmov s0, w8
; CHECK-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_i64_bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    cmp x0, #0
; CHECK-FP16-NEXT:    cneg x8, x0, mi
; CHECK-FP16-NEXT:    lsr x9, x8, #53
; CHECK-FP16-NEXT:    and x10, x8, #0xfffffffffffff000
; CHECK-FP16-NEXT:    cmp x9, #0
; CHECK-FP16-NEXT:    csel x9, x10, x8, ne
; CHECK-FP16-NEXT:    and x10, x0, #0x8000000000000000
; CHECK-FP16-NEXT:    cset w11, ne
; CHECK-FP16-NEXT:    scvtf d0, x9
; CHECK-FP16-NEXT:    tst x8, #0xfff
; CHECK-FP16-NEXT:    fmov x9, d0
; CHECK-FP16-NEXT:    orr x8, x9, x10
; CHECK-FP16-NEXT:    csel w9, wzr, w11, eq
; CHECK-FP16-NEXT:    orr x8, x8, x9
; CHECK-FP16-NEXT:    fmov d0, x8
; CHECK-FP16-NEXT:    fcvtxn s0, d0
; CHECK-FP16-NEXT:    bfcvt h0, s0
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp i64 %a to bfloat
  ret bfloat %c
}

define bfloat @utofp_i64_bf16(i64 %a) {
; CHECK-NOFP16-LABEL: utofp_i64_bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    lsr x9, x0, #53
; CHECK-NOFP16-NEXT:    mov w8, #32767 // =0x7fff
; CHECK-NOFP16-NEXT:    cmp x9, #0
; CHECK-NOFP16-NEXT:    and x9, x0, #0xfffffffffffff000
; CHECK-NOFP16-NEXT:    csel x9, x9, x0, ne
; CHECK-NOFP16-NEXT:    ucvtf d0, x9
; CHECK-NOFP16-NEXT:    cset w9, ne
; CHECK-NOFP16-NEXT:    tst x0, #0xfff
; CHECK-NOFP16-NEXT:    csel w9, wzr, w9, eq
; CHECK-NOFP16-NEXT:    fmov x10, d0
; CHECK-NOFP16-NEXT:    orr x9, x10, x9
; CHECK-NOFP16-NEXT:    fmov d0, x9
; CHECK-NOFP16-NEXT:    fcvtxn s0, d0
; CHECK-NOFP16-NEXT:    fmov w9, s0
; CHECK-NOFP16-NEXT:    ubfx w10, w9, #16, #1
; CHECK-NOFP16-NEXT:    add w8, w9, w8
; CHECK-NOFP16-NEXT:    add w8, w10, w8
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    fmov s0, w8
; CHECK-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_i64_bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    lsr x8, x0, #53
; CHECK-FP16-NEXT:    and x9, x0, #0xfffffffffffff000
; CHECK-FP16-NEXT:    cmp x8, #0
; CHECK-FP16-NEXT:    csel x8, x9, x0, ne
; CHECK-FP16-NEXT:    ucvtf d0, x8
; CHECK-FP16-NEXT:    cset w8, ne
; CHECK-FP16-NEXT:    tst x0, #0xfff
; CHECK-FP16-NEXT:    csel w8, wzr, w8, eq
; CHECK-FP16-NEXT:    fmov x9, d0
; CHECK-FP16-NEXT:    orr x8, x9, x8
; CHECK-FP16-NEXT:    fmov d0, x8
; CHECK-FP16-NEXT:    fcvtxn s0, d0
; CHECK-FP16-NEXT:    bfcvt h0, s0
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp i64 %a to bfloat
  ret bfloat %c
}

define bfloat @stofp_i32_bf16(i32 %a) {
; CHECK-NOFP16-LABEL: stofp_i32_bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    scvtf d0, w0
; CHECK-NOFP16-NEXT:    mov w8, #32767 // =0x7fff
; CHECK-NOFP16-NEXT:    fcvtxn s0, d0
; CHECK-NOFP16-NEXT:    fmov w9, s0
; CHECK-NOFP16-NEXT:    ubfx w10, w9, #16, #1
; CHECK-NOFP16-NEXT:    add w8, w9, w8
; CHECK-NOFP16-NEXT:    add w8, w10, w8
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    fmov s0, w8
; CHECK-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_i32_bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    scvtf d0, w0
; CHECK-FP16-NEXT:    fcvtxn s0, d0
; CHECK-FP16-NEXT:    bfcvt h0, s0
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp i32 %a to bfloat
  ret bfloat %c
}

define bfloat @utofp_i32_bf16(i32 %a) {
; CHECK-NOFP16-LABEL: utofp_i32_bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ucvtf d0, w0
; CHECK-NOFP16-NEXT:    mov w8, #32767 // =0x7fff
; CHECK-NOFP16-NEXT:    fcvtxn s0, d0
; CHECK-NOFP16-NEXT:    fmov w9, s0
; CHECK-NOFP16-NEXT:    ubfx w10, w9, #16, #1
; CHECK-NOFP16-NEXT:    add w8, w9, w8
; CHECK-NOFP16-NEXT:    add w8, w10, w8
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    fmov s0, w8
; CHECK-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_i32_bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ucvtf d0, w0
; CHECK-FP16-NEXT:    fcvtxn s0, d0
; CHECK-FP16-NEXT:    bfcvt h0, s0
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp i32 %a to bfloat
  ret bfloat %c
}

define bfloat @stofp_i16_bf16(i16 %a) {
; CHECK-NOFP16-LABEL: stofp_i16_bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    sxth w9, w0
; CHECK-NOFP16-NEXT:    mov w8, #32767 // =0x7fff
; CHECK-NOFP16-NEXT:    scvtf s0, w9
; CHECK-NOFP16-NEXT:    fmov w9, s0
; CHECK-NOFP16-NEXT:    ubfx w10, w9, #16, #1
; CHECK-NOFP16-NEXT:    add w8, w9, w8
; CHECK-NOFP16-NEXT:    add w8, w10, w8
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    fmov s0, w8
; CHECK-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_i16_bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    sxth w8, w0
; CHECK-FP16-NEXT:    scvtf s0, w8
; CHECK-FP16-NEXT:    bfcvt h0, s0
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp i16 %a to bfloat
  ret bfloat %c
}

define bfloat @utofp_i16_bf16(i16 %a) {
; CHECK-NOFP16-LABEL: utofp_i16_bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    and w9, w0, #0xffff
; CHECK-NOFP16-NEXT:    mov w8, #32767 // =0x7fff
; CHECK-NOFP16-NEXT:    ucvtf s0, w9
; CHECK-NOFP16-NEXT:    fmov w9, s0
; CHECK-NOFP16-NEXT:    ubfx w10, w9, #16, #1
; CHECK-NOFP16-NEXT:    add w8, w9, w8
; CHECK-NOFP16-NEXT:    add w8, w10, w8
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    fmov s0, w8
; CHECK-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_i16_bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    and w8, w0, #0xffff
; CHECK-FP16-NEXT:    ucvtf s0, w8
; CHECK-FP16-NEXT:    bfcvt h0, s0
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp i16 %a to bfloat
  ret bfloat %c
}

define bfloat @stofp_i8_bf16(i8 %a) {
; CHECK-NOFP16-LABEL: stofp_i8_bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    sxtb w9, w0
; CHECK-NOFP16-NEXT:    mov w8, #32767 // =0x7fff
; CHECK-NOFP16-NEXT:    scvtf s0, w9
; CHECK-NOFP16-NEXT:    fmov w9, s0
; CHECK-NOFP16-NEXT:    ubfx w10, w9, #16, #1
; CHECK-NOFP16-NEXT:    add w8, w9, w8
; CHECK-NOFP16-NEXT:    add w8, w10, w8
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    fmov s0, w8
; CHECK-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_i8_bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    sxtb w8, w0
; CHECK-FP16-NEXT:    scvtf s0, w8
; CHECK-FP16-NEXT:    bfcvt h0, s0
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp i8 %a to bfloat
  ret bfloat %c
}

define bfloat @utofp_i8_bf16(i8 %a) {
; CHECK-NOFP16-LABEL: utofp_i8_bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    and w9, w0, #0xff
; CHECK-NOFP16-NEXT:    mov w8, #32767 // =0x7fff
; CHECK-NOFP16-NEXT:    ucvtf s0, w9
; CHECK-NOFP16-NEXT:    fmov w9, s0
; CHECK-NOFP16-NEXT:    ubfx w10, w9, #16, #1
; CHECK-NOFP16-NEXT:    add w8, w9, w8
; CHECK-NOFP16-NEXT:    add w8, w10, w8
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    fmov s0, w8
; CHECK-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_i8_bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    and w8, w0, #0xff
; CHECK-FP16-NEXT:    ucvtf s0, w8
; CHECK-FP16-NEXT:    bfcvt h0, s0
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp i8 %a to bfloat
  ret bfloat %c
}

define <2 x bfloat> @stofp_v2i64_v2bf16(<2 x i64> %a) {
; CHECK-NOFP16-LABEL: stofp_v2i64_v2bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    mov x9, v0.d[1]
; CHECK-NOFP16-NEXT:    mov w8, #32767 // =0x7fff
; CHECK-NOFP16-NEXT:    cmp x9, #0
; CHECK-NOFP16-NEXT:    cneg x10, x9, mi
; CHECK-NOFP16-NEXT:    and x9, x9, #0x8000000000000000
; CHECK-NOFP16-NEXT:    lsr x11, x10, #53
; CHECK-NOFP16-NEXT:    and x12, x10, #0xfffffffffffff000
; CHECK-NOFP16-NEXT:    cmp x11, #0
; CHECK-NOFP16-NEXT:    csel x11, x12, x10, ne
; CHECK-NOFP16-NEXT:    cset w12, ne
; CHECK-NOFP16-NEXT:    tst x10, #0xfff
; CHECK-NOFP16-NEXT:    fmov x10, d0
; CHECK-NOFP16-NEXT:    csel w12, wzr, w12, eq
; CHECK-NOFP16-NEXT:    scvtf d0, x11
; CHECK-NOFP16-NEXT:    cmp x10, #0
; CHECK-NOFP16-NEXT:    cneg x13, x10, mi
; CHECK-NOFP16-NEXT:    and x10, x10, #0x8000000000000000
; CHECK-NOFP16-NEXT:    lsr x14, x13, #53
; CHECK-NOFP16-NEXT:    cmp x14, #0
; CHECK-NOFP16-NEXT:    and x14, x13, #0xfffffffffffff000
; CHECK-NOFP16-NEXT:    csel x11, x14, x13, ne
; CHECK-NOFP16-NEXT:    cset w14, ne
; CHECK-NOFP16-NEXT:    tst x13, #0xfff
; CHECK-NOFP16-NEXT:    scvtf d1, x11
; CHECK-NOFP16-NEXT:    fmov x11, d0
; CHECK-NOFP16-NEXT:    orr x9, x11, x9
; CHECK-NOFP16-NEXT:    csel w11, wzr, w14, eq
; CHECK-NOFP16-NEXT:    fmov x13, d1
; CHECK-NOFP16-NEXT:    orr x9, x9, x12
; CHECK-NOFP16-NEXT:    fmov d0, x9
; CHECK-NOFP16-NEXT:    orr x10, x13, x10
; CHECK-NOFP16-NEXT:    orr x10, x10, x11
; CHECK-NOFP16-NEXT:    fcvtxn s0, d0
; CHECK-NOFP16-NEXT:    fmov d1, x10
; CHECK-NOFP16-NEXT:    fcvtxn s1, d1
; CHECK-NOFP16-NEXT:    fmov w9, s0
; CHECK-NOFP16-NEXT:    ubfx w11, w9, #16, #1
; CHECK-NOFP16-NEXT:    add w9, w9, w8
; CHECK-NOFP16-NEXT:    fmov w10, s1
; CHECK-NOFP16-NEXT:    add w9, w11, w9
; CHECK-NOFP16-NEXT:    lsr w9, w9, #16
; CHECK-NOFP16-NEXT:    ubfx w12, w10, #16, #1
; CHECK-NOFP16-NEXT:    add w8, w10, w8
; CHECK-NOFP16-NEXT:    fmov s1, w9
; CHECK-NOFP16-NEXT:    add w8, w12, w8
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    fmov s0, w8
; CHECK-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v2i64_v2bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    mov x8, v0.d[1]
; CHECK-FP16-NEXT:    cmp x8, #0
; CHECK-FP16-NEXT:    cneg x9, x8, mi
; CHECK-FP16-NEXT:    and x8, x8, #0x8000000000000000
; CHECK-FP16-NEXT:    lsr x10, x9, #53
; CHECK-FP16-NEXT:    and x11, x9, #0xfffffffffffff000
; CHECK-FP16-NEXT:    cmp x10, #0
; CHECK-FP16-NEXT:    csel x10, x11, x9, ne
; CHECK-FP16-NEXT:    cset w11, ne
; CHECK-FP16-NEXT:    tst x9, #0xfff
; CHECK-FP16-NEXT:    fmov x9, d0
; CHECK-FP16-NEXT:    csel w11, wzr, w11, eq
; CHECK-FP16-NEXT:    scvtf d0, x10
; CHECK-FP16-NEXT:    cmp x9, #0
; CHECK-FP16-NEXT:    cneg x12, x9, mi
; CHECK-FP16-NEXT:    and x9, x9, #0x8000000000000000
; CHECK-FP16-NEXT:    lsr x13, x12, #53
; CHECK-FP16-NEXT:    cmp x13, #0
; CHECK-FP16-NEXT:    and x13, x12, #0xfffffffffffff000
; CHECK-FP16-NEXT:    csel x10, x13, x12, ne
; CHECK-FP16-NEXT:    cset w13, ne
; CHECK-FP16-NEXT:    tst x12, #0xfff
; CHECK-FP16-NEXT:    scvtf d1, x10
; CHECK-FP16-NEXT:    fmov x10, d0
; CHECK-FP16-NEXT:    orr x8, x10, x8
; CHECK-FP16-NEXT:    csel w10, wzr, w13, eq
; CHECK-FP16-NEXT:    fmov x12, d1
; CHECK-FP16-NEXT:    orr x8, x8, x11
; CHECK-FP16-NEXT:    fmov d0, x8
; CHECK-FP16-NEXT:    orr x9, x12, x9
; CHECK-FP16-NEXT:    orr x9, x9, x10
; CHECK-FP16-NEXT:    fcvtxn s0, d0
; CHECK-FP16-NEXT:    fmov d1, x9
; CHECK-FP16-NEXT:    fcvtxn s1, d1
; CHECK-FP16-NEXT:    bfcvt h2, s0
; CHECK-FP16-NEXT:    bfcvt h0, s1
; CHECK-FP16-NEXT:    mov v0.h[1], v2.h[0]
; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <2 x i64> %a to <2 x bfloat>
  ret <2 x bfloat> %c
}

define <2 x bfloat> @utofp_v2i64_v2bf16(<2 x i64> %a) {
; CHECK-NOFP16-LABEL: utofp_v2i64_v2bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    mov x9, v0.d[1]
; CHECK-NOFP16-NEXT:    fmov x11, d0
; CHECK-NOFP16-NEXT:    mov w8, #32767 // =0x7fff
; CHECK-NOFP16-NEXT:    lsr x10, x9, #53
; CHECK-NOFP16-NEXT:    and x12, x9, #0xfffffffffffff000
; CHECK-NOFP16-NEXT:    cmp x10, #0
; CHECK-NOFP16-NEXT:    lsr x10, x11, #53
; CHECK-NOFP16-NEXT:    csel x12, x12, x9, ne
; CHECK-NOFP16-NEXT:    cset w13, ne
; CHECK-NOFP16-NEXT:    tst x9, #0xfff
; CHECK-NOFP16-NEXT:    csel w9, wzr, w13, eq
; CHECK-NOFP16-NEXT:    cmp x10, #0
; CHECK-NOFP16-NEXT:    and x10, x11, #0xfffffffffffff000
; CHECK-NOFP16-NEXT:    csel x10, x10, x11, ne
; CHECK-NOFP16-NEXT:    ucvtf d0, x12
; CHECK-NOFP16-NEXT:    ucvtf d1, x10
; CHECK-NOFP16-NEXT:    cset w10, ne
; CHECK-NOFP16-NEXT:    tst x11, #0xfff
; CHECK-NOFP16-NEXT:    csel w10, wzr, w10, eq
; CHECK-NOFP16-NEXT:    fmov x11, d0
; CHECK-NOFP16-NEXT:    fmov x12, d1
; CHECK-NOFP16-NEXT:    orr x9, x11, x9
; CHECK-NOFP16-NEXT:    orr x10, x12, x10
; CHECK-NOFP16-NEXT:    fmov d0, x9
; CHECK-NOFP16-NEXT:    fmov d1, x10
; CHECK-NOFP16-NEXT:    fcvtxn s0, d0
; CHECK-NOFP16-NEXT:    fcvtxn s1, d1
; CHECK-NOFP16-NEXT:    fmov w9, s0
; CHECK-NOFP16-NEXT:    fmov w10, s1
; CHECK-NOFP16-NEXT:    ubfx w11, w9, #16, #1
; CHECK-NOFP16-NEXT:    add w9, w9, w8
; CHECK-NOFP16-NEXT:    ubfx w12, w10, #16, #1
; CHECK-NOFP16-NEXT:    add w8, w10, w8
; CHECK-NOFP16-NEXT:    add w9, w11, w9
; CHECK-NOFP16-NEXT:    add w8, w12, w8
; CHECK-NOFP16-NEXT:    lsr w9, w9, #16
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    fmov s1, w9
; CHECK-NOFP16-NEXT:    fmov s0, w8
; CHECK-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v2i64_v2bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    mov x8, v0.d[1]
; CHECK-FP16-NEXT:    fmov x10, d0
; CHECK-FP16-NEXT:    lsr x9, x8, #53
; CHECK-FP16-NEXT:    and x11, x8, #0xfffffffffffff000
; CHECK-FP16-NEXT:    cmp x9, #0
; CHECK-FP16-NEXT:    lsr x9, x10, #53
; CHECK-FP16-NEXT:    csel x11, x11, x8, ne
; CHECK-FP16-NEXT:    cset w12, ne
; CHECK-FP16-NEXT:    tst x8, #0xfff
; CHECK-FP16-NEXT:    csel w8, wzr, w12, eq
; CHECK-FP16-NEXT:    cmp x9, #0
; CHECK-FP16-NEXT:    and x9, x10, #0xfffffffffffff000
; CHECK-FP16-NEXT:    csel x9, x9, x10, ne
; CHECK-FP16-NEXT:    ucvtf d0, x11
; CHECK-FP16-NEXT:    ucvtf d1, x9
; CHECK-FP16-NEXT:    cset w9, ne
; CHECK-FP16-NEXT:    tst x10, #0xfff
; CHECK-FP16-NEXT:    csel w9, wzr, w9, eq
; CHECK-FP16-NEXT:    fmov x10, d0
; CHECK-FP16-NEXT:    fmov x11, d1
; CHECK-FP16-NEXT:    orr x8, x10, x8
; CHECK-FP16-NEXT:    orr x9, x11, x9
; CHECK-FP16-NEXT:    fmov d0, x8
; CHECK-FP16-NEXT:    fmov d1, x9
; CHECK-FP16-NEXT:    fcvtxn s0, d0
; CHECK-FP16-NEXT:    fcvtxn s1, d1
; CHECK-FP16-NEXT:    bfcvt h2, s0
; CHECK-FP16-NEXT:    bfcvt h0, s1
; CHECK-FP16-NEXT:    mov v0.h[1], v2.h[0]
; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <2 x i64> %a to <2 x bfloat>
  ret <2 x bfloat> %c
}

define <3 x bfloat> @stofp_v3i64_v3bf16(<3 x i64> %a) {
; CHECK-NOFP16-LABEL: stofp_v3i64_v3bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-NOFP16-NEXT:    fmov x8, d0
; CHECK-NOFP16-NEXT:    movi v3.2d, #0000000000000000
; CHECK-NOFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-NOFP16-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-NOFP16-NEXT:    scvtf s0, x8
; CHECK-NOFP16-NEXT:    fmov x8, d1
; CHECK-NOFP16-NEXT:    scvtf s1, x8
; CHECK-NOFP16-NEXT:    fmov x8, d2
; CHECK-NOFP16-NEXT:    mov v3.s[0], v0.s[0]
; CHECK-NOFP16-NEXT:    scvtf s0, x8
; CHECK-NOFP16-NEXT:    mov v3.s[1], v1.s[0]
; CHECK-NOFP16-NEXT:    movi v1.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    mov v3.s[2], v0.s[0]
; CHECK-NOFP16-NEXT:    movi v0.4s, #1
; CHECK-NOFP16-NEXT:    ushr v2.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    add v1.4s, v3.4s, v1.4s
; CHECK-NOFP16-NEXT:    and v0.16b, v2.16b, v0.16b
; CHECK-NOFP16-NEXT:    fcmeq v2.4s, v3.4s, v3.4s
; CHECK-NOFP16-NEXT:    orr v3.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-NOFP16-NEXT:    bif v0.16b, v3.16b, v2.16b
; CHECK-NOFP16-NEXT:    shrn v0.4h, v0.4s, #16
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v3i64_v3bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-FP16-NEXT:    fmov x8, d0
; CHECK-FP16-NEXT:    movi v3.2d, #0000000000000000
; CHECK-FP16-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-FP16-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-FP16-NEXT:    scvtf s0, x8
; CHECK-FP16-NEXT:    fmov x8, d1
; CHECK-FP16-NEXT:    scvtf s1, x8
; CHECK-FP16-NEXT:    fmov x8, d2
; CHECK-FP16-NEXT:    mov v3.s[0], v0.s[0]
; CHECK-FP16-NEXT:    scvtf s0, x8
; CHECK-FP16-NEXT:    mov v3.s[1], v1.s[0]
; CHECK-FP16-NEXT:    mov v3.s[2], v0.s[0]
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v3.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <3 x i64> %a to <3 x bfloat>
  ret <3 x bfloat> %c
}

define <3 x bfloat> @utofp_v3i64_v3bf16(<3 x i64> %a) {
; CHECK-NOFP16-LABEL: utofp_v3i64_v3bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-NOFP16-NEXT:    fmov x8, d0
; CHECK-NOFP16-NEXT:    movi v3.2d, #0000000000000000
; CHECK-NOFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-NOFP16-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-NOFP16-NEXT:    ucvtf s0, x8
; CHECK-NOFP16-NEXT:    fmov x8, d1
; CHECK-NOFP16-NEXT:    ucvtf s1, x8
; CHECK-NOFP16-NEXT:    fmov x8, d2
; CHECK-NOFP16-NEXT:    mov v3.s[0], v0.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s0, x8
; CHECK-NOFP16-NEXT:    mov v3.s[1], v1.s[0]
; CHECK-NOFP16-NEXT:    movi v1.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    mov v3.s[2], v0.s[0]
; CHECK-NOFP16-NEXT:    movi v0.4s, #1
; CHECK-NOFP16-NEXT:    ushr v2.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    add v1.4s, v3.4s, v1.4s
; CHECK-NOFP16-NEXT:    and v0.16b, v2.16b, v0.16b
; CHECK-NOFP16-NEXT:    fcmeq v2.4s, v3.4s, v3.4s
; CHECK-NOFP16-NEXT:    orr v3.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-NOFP16-NEXT:    bif v0.16b, v3.16b, v2.16b
; CHECK-NOFP16-NEXT:    shrn v0.4h, v0.4s, #16
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v3i64_v3bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-FP16-NEXT:    fmov x8, d0
; CHECK-FP16-NEXT:    movi v3.2d, #0000000000000000
; CHECK-FP16-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-FP16-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-FP16-NEXT:    ucvtf s0, x8
; CHECK-FP16-NEXT:    fmov x8, d1
; CHECK-FP16-NEXT:    ucvtf s1, x8
; CHECK-FP16-NEXT:    fmov x8, d2
; CHECK-FP16-NEXT:    mov v3.s[0], v0.s[0]
; CHECK-FP16-NEXT:    ucvtf s0, x8
; CHECK-FP16-NEXT:    mov v3.s[1], v1.s[0]
; CHECK-FP16-NEXT:    mov v3.s[2], v0.s[0]
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v3.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <3 x i64> %a to <3 x bfloat>
  ret <3 x bfloat> %c
}

define <4 x bfloat> @stofp_v4i64_v4bf16(<4 x i64> %a) {
; CHECK-NOFP16-LABEL: stofp_v4i64_v4bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    mov x8, v0.d[1]
; CHECK-NOFP16-NEXT:    fmov x9, d0
; CHECK-NOFP16-NEXT:    scvtf s2, x9
; CHECK-NOFP16-NEXT:    mov x9, v1.d[1]
; CHECK-NOFP16-NEXT:    scvtf s0, x8
; CHECK-NOFP16-NEXT:    fmov x8, d1
; CHECK-NOFP16-NEXT:    scvtf s1, x8
; CHECK-NOFP16-NEXT:    mov v2.s[1], v0.s[0]
; CHECK-NOFP16-NEXT:    scvtf s0, x9
; CHECK-NOFP16-NEXT:    mov v2.s[2], v1.s[0]
; CHECK-NOFP16-NEXT:    movi v1.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    mov v2.s[3], v0.s[0]
; CHECK-NOFP16-NEXT:    movi v0.4s, #1
; CHECK-NOFP16-NEXT:    ushr v3.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    add v1.4s, v2.4s, v1.4s
; CHECK-NOFP16-NEXT:    and v0.16b, v3.16b, v0.16b
; CHECK-NOFP16-NEXT:    fcmeq v3.4s, v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    orr v2.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-NOFP16-NEXT:    bif v0.16b, v2.16b, v3.16b
; CHECK-NOFP16-NEXT:    shrn v0.4h, v0.4s, #16
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v4i64_v4bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    mov x8, v0.d[1]
; CHECK-FP16-NEXT:    fmov x9, d0
; CHECK-FP16-NEXT:    scvtf s2, x9
; CHECK-FP16-NEXT:    mov x9, v1.d[1]
; CHECK-FP16-NEXT:    scvtf s0, x8
; CHECK-FP16-NEXT:    fmov x8, d1
; CHECK-FP16-NEXT:    mov v2.s[1], v0.s[0]
; CHECK-FP16-NEXT:    scvtf s0, x8
; CHECK-FP16-NEXT:    mov v2.s[2], v0.s[0]
; CHECK-FP16-NEXT:    scvtf s0, x9
; CHECK-FP16-NEXT:    mov v2.s[3], v0.s[0]
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v2.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <4 x i64> %a to <4 x bfloat>
  ret <4 x bfloat> %c
}

define <4 x bfloat> @utofp_v4i64_v4bf16(<4 x i64> %a) {
; CHECK-NOFP16-LABEL: utofp_v4i64_v4bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    mov x8, v0.d[1]
; CHECK-NOFP16-NEXT:    fmov x9, d0
; CHECK-NOFP16-NEXT:    ucvtf s2, x9
; CHECK-NOFP16-NEXT:    mov x9, v1.d[1]
; CHECK-NOFP16-NEXT:    ucvtf s0, x8
; CHECK-NOFP16-NEXT:    fmov x8, d1
; CHECK-NOFP16-NEXT:    ucvtf s1, x8
; CHECK-NOFP16-NEXT:    mov v2.s[1], v0.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s0, x9
; CHECK-NOFP16-NEXT:    mov v2.s[2], v1.s[0]
; CHECK-NOFP16-NEXT:    movi v1.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    mov v2.s[3], v0.s[0]
; CHECK-NOFP16-NEXT:    movi v0.4s, #1
; CHECK-NOFP16-NEXT:    ushr v3.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    add v1.4s, v2.4s, v1.4s
; CHECK-NOFP16-NEXT:    and v0.16b, v3.16b, v0.16b
; CHECK-NOFP16-NEXT:    fcmeq v3.4s, v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    orr v2.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-NOFP16-NEXT:    bif v0.16b, v2.16b, v3.16b
; CHECK-NOFP16-NEXT:    shrn v0.4h, v0.4s, #16
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v4i64_v4bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    mov x8, v0.d[1]
; CHECK-FP16-NEXT:    fmov x9, d0
; CHECK-FP16-NEXT:    ucvtf s2, x9
; CHECK-FP16-NEXT:    mov x9, v1.d[1]
; CHECK-FP16-NEXT:    ucvtf s0, x8
; CHECK-FP16-NEXT:    fmov x8, d1
; CHECK-FP16-NEXT:    mov v2.s[1], v0.s[0]
; CHECK-FP16-NEXT:    ucvtf s0, x8
; CHECK-FP16-NEXT:    mov v2.s[2], v0.s[0]
; CHECK-FP16-NEXT:    ucvtf s0, x9
; CHECK-FP16-NEXT:    mov v2.s[3], v0.s[0]
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v2.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <4 x i64> %a to <4 x bfloat>
  ret <4 x bfloat> %c
}

define <8 x bfloat> @stofp_v8i64_v8bf16(<8 x i64> %a) {
; CHECK-NOFP16-LABEL: stofp_v8i64_v8bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    fmov x10, d2
; CHECK-NOFP16-NEXT:    mov x8, v2.d[1]
; CHECK-NOFP16-NEXT:    mov x9, v0.d[1]
; CHECK-NOFP16-NEXT:    scvtf s2, x10
; CHECK-NOFP16-NEXT:    fmov x10, d0
; CHECK-NOFP16-NEXT:    scvtf s0, x8
; CHECK-NOFP16-NEXT:    scvtf s5, x9
; CHECK-NOFP16-NEXT:    fmov x9, d3
; CHECK-NOFP16-NEXT:    mov x8, v3.d[1]
; CHECK-NOFP16-NEXT:    scvtf s4, x10
; CHECK-NOFP16-NEXT:    fmov x10, d1
; CHECK-NOFP16-NEXT:    scvtf s3, x9
; CHECK-NOFP16-NEXT:    mov x9, v1.d[1]
; CHECK-NOFP16-NEXT:    mov v2.s[1], v0.s[0]
; CHECK-NOFP16-NEXT:    scvtf s0, x10
; CHECK-NOFP16-NEXT:    scvtf s1, x8
; CHECK-NOFP16-NEXT:    mov v4.s[1], v5.s[0]
; CHECK-NOFP16-NEXT:    mov v2.s[2], v3.s[0]
; CHECK-NOFP16-NEXT:    scvtf s3, x9
; CHECK-NOFP16-NEXT:    mov v4.s[2], v0.s[0]
; CHECK-NOFP16-NEXT:    movi v0.4s, #1
; CHECK-NOFP16-NEXT:    mov v2.s[3], v1.s[0]
; CHECK-NOFP16-NEXT:    movi v1.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    mov v4.s[3], v3.s[0]
; CHECK-NOFP16-NEXT:    ushr v3.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    add v6.4s, v2.4s, v1.4s
; CHECK-NOFP16-NEXT:    ushr v5.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    add v1.4s, v4.4s, v1.4s
; CHECK-NOFP16-NEXT:    and v3.16b, v3.16b, v0.16b
; CHECK-NOFP16-NEXT:    and v0.16b, v5.16b, v0.16b
; CHECK-NOFP16-NEXT:    fcmeq v5.4s, v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    orr v2.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v3.4s, v3.4s, v6.4s
; CHECK-NOFP16-NEXT:    fcmeq v6.4s, v4.4s, v4.4s
; CHECK-NOFP16-NEXT:    orr v4.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-NOFP16-NEXT:    mov v1.16b, v5.16b
; CHECK-NOFP16-NEXT:    bif v0.16b, v4.16b, v6.16b
; CHECK-NOFP16-NEXT:    bsl v1.16b, v3.16b, v2.16b
; CHECK-NOFP16-NEXT:    uzp2 v0.8h, v0.8h, v1.8h
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v8i64_v8bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    mov x9, v0.d[1]
; CHECK-FP16-NEXT:    fmov x10, d0
; CHECK-FP16-NEXT:    mov x8, v2.d[1]
; CHECK-FP16-NEXT:    scvtf s4, x10
; CHECK-FP16-NEXT:    fmov x10, d1
; CHECK-FP16-NEXT:    scvtf s0, x9
; CHECK-FP16-NEXT:    fmov x9, d2
; CHECK-FP16-NEXT:    scvtf s2, x8
; CHECK-FP16-NEXT:    mov x8, v1.d[1]
; CHECK-FP16-NEXT:    scvtf s1, x9
; CHECK-FP16-NEXT:    fmov x9, d3
; CHECK-FP16-NEXT:    mov v4.s[1], v0.s[0]
; CHECK-FP16-NEXT:    scvtf s0, x10
; CHECK-FP16-NEXT:    mov x10, v3.d[1]
; CHECK-FP16-NEXT:    scvtf s3, x9
; CHECK-FP16-NEXT:    mov v1.s[1], v2.s[0]
; CHECK-FP16-NEXT:    scvtf s2, x8
; CHECK-FP16-NEXT:    mov v4.s[2], v0.s[0]
; CHECK-FP16-NEXT:    scvtf s0, x10
; CHECK-FP16-NEXT:    mov v1.s[2], v3.s[0]
; CHECK-FP16-NEXT:    mov v4.s[3], v2.s[0]
; CHECK-FP16-NEXT:    mov v1.s[3], v0.s[0]
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v4.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v1.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <8 x i64> %a to <8 x bfloat>
  ret <8 x bfloat> %c
}

define <8 x bfloat> @utofp_v8i64_v8bf16(<8 x i64> %a) {
; CHECK-NOFP16-LABEL: utofp_v8i64_v8bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    fmov x10, d2
; CHECK-NOFP16-NEXT:    mov x8, v2.d[1]
; CHECK-NOFP16-NEXT:    mov x9, v0.d[1]
; CHECK-NOFP16-NEXT:    ucvtf s2, x10
; CHECK-NOFP16-NEXT:    fmov x10, d0
; CHECK-NOFP16-NEXT:    ucvtf s0, x8
; CHECK-NOFP16-NEXT:    ucvtf s5, x9
; CHECK-NOFP16-NEXT:    fmov x9, d3
; CHECK-NOFP16-NEXT:    mov x8, v3.d[1]
; CHECK-NOFP16-NEXT:    ucvtf s4, x10
; CHECK-NOFP16-NEXT:    fmov x10, d1
; CHECK-NOFP16-NEXT:    ucvtf s3, x9
; CHECK-NOFP16-NEXT:    mov x9, v1.d[1]
; CHECK-NOFP16-NEXT:    mov v2.s[1], v0.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s0, x10
; CHECK-NOFP16-NEXT:    ucvtf s1, x8
; CHECK-NOFP16-NEXT:    mov v4.s[1], v5.s[0]
; CHECK-NOFP16-NEXT:    mov v2.s[2], v3.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s3, x9
; CHECK-NOFP16-NEXT:    mov v4.s[2], v0.s[0]
; CHECK-NOFP16-NEXT:    movi v0.4s, #1
; CHECK-NOFP16-NEXT:    mov v2.s[3], v1.s[0]
; CHECK-NOFP16-NEXT:    movi v1.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    mov v4.s[3], v3.s[0]
; CHECK-NOFP16-NEXT:    ushr v3.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    add v6.4s, v2.4s, v1.4s
; CHECK-NOFP16-NEXT:    ushr v5.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    add v1.4s, v4.4s, v1.4s
; CHECK-NOFP16-NEXT:    and v3.16b, v3.16b, v0.16b
; CHECK-NOFP16-NEXT:    and v0.16b, v5.16b, v0.16b
; CHECK-NOFP16-NEXT:    fcmeq v5.4s, v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    orr v2.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v3.4s, v3.4s, v6.4s
; CHECK-NOFP16-NEXT:    fcmeq v6.4s, v4.4s, v4.4s
; CHECK-NOFP16-NEXT:    orr v4.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-NOFP16-NEXT:    mov v1.16b, v5.16b
; CHECK-NOFP16-NEXT:    bif v0.16b, v4.16b, v6.16b
; CHECK-NOFP16-NEXT:    bsl v1.16b, v3.16b, v2.16b
; CHECK-NOFP16-NEXT:    uzp2 v0.8h, v0.8h, v1.8h
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v8i64_v8bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    mov x9, v0.d[1]
; CHECK-FP16-NEXT:    fmov x10, d0
; CHECK-FP16-NEXT:    mov x8, v2.d[1]
; CHECK-FP16-NEXT:    ucvtf s4, x10
; CHECK-FP16-NEXT:    fmov x10, d1
; CHECK-FP16-NEXT:    ucvtf s0, x9
; CHECK-FP16-NEXT:    fmov x9, d2
; CHECK-FP16-NEXT:    ucvtf s2, x8
; CHECK-FP16-NEXT:    mov x8, v1.d[1]
; CHECK-FP16-NEXT:    ucvtf s1, x9
; CHECK-FP16-NEXT:    fmov x9, d3
; CHECK-FP16-NEXT:    mov v4.s[1], v0.s[0]
; CHECK-FP16-NEXT:    ucvtf s0, x10
; CHECK-FP16-NEXT:    mov x10, v3.d[1]
; CHECK-FP16-NEXT:    ucvtf s3, x9
; CHECK-FP16-NEXT:    mov v1.s[1], v2.s[0]
; CHECK-FP16-NEXT:    ucvtf s2, x8
; CHECK-FP16-NEXT:    mov v4.s[2], v0.s[0]
; CHECK-FP16-NEXT:    ucvtf s0, x10
; CHECK-FP16-NEXT:    mov v1.s[2], v3.s[0]
; CHECK-FP16-NEXT:    mov v4.s[3], v2.s[0]
; CHECK-FP16-NEXT:    mov v1.s[3], v0.s[0]
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v4.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v1.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <8 x i64> %a to <8 x bfloat>
  ret <8 x bfloat> %c
}

define <16 x bfloat> @stofp_v16i64_v16bf16(<16 x i64> %a) {
; CHECK-NOFP16-LABEL: stofp_v16i64_v16bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    mov x8, v2.d[1]
; CHECK-NOFP16-NEXT:    fmov x11, d0
; CHECK-NOFP16-NEXT:    mov x9, v0.d[1]
; CHECK-NOFP16-NEXT:    fmov x10, d2
; CHECK-NOFP16-NEXT:    mov x12, v6.d[1]
; CHECK-NOFP16-NEXT:    scvtf s2, x11
; CHECK-NOFP16-NEXT:    fmov x11, d3
; CHECK-NOFP16-NEXT:    scvtf s16, x8
; CHECK-NOFP16-NEXT:    fmov x8, d6
; CHECK-NOFP16-NEXT:    scvtf s0, x10
; CHECK-NOFP16-NEXT:    mov x10, v4.d[1]
; CHECK-NOFP16-NEXT:    scvtf s17, x9
; CHECK-NOFP16-NEXT:    mov x9, v3.d[1]
; CHECK-NOFP16-NEXT:    scvtf s6, x12
; CHECK-NOFP16-NEXT:    fmov x12, d4
; CHECK-NOFP16-NEXT:    scvtf s4, x11
; CHECK-NOFP16-NEXT:    scvtf s3, x8
; CHECK-NOFP16-NEXT:    fmov x11, d7
; CHECK-NOFP16-NEXT:    mov x8, v1.d[1]
; CHECK-NOFP16-NEXT:    mov v0.s[1], v16.s[0]
; CHECK-NOFP16-NEXT:    scvtf s18, x10
; CHECK-NOFP16-NEXT:    scvtf s19, x12
; CHECK-NOFP16-NEXT:    fmov x10, d1
; CHECK-NOFP16-NEXT:    mov v2.s[1], v17.s[0]
; CHECK-NOFP16-NEXT:    mov x12, v5.d[1]
; CHECK-NOFP16-NEXT:    mov v3.s[1], v6.s[0]
; CHECK-NOFP16-NEXT:    scvtf s6, x11
; CHECK-NOFP16-NEXT:    fmov x11, d5
; CHECK-NOFP16-NEXT:    scvtf s1, x10
; CHECK-NOFP16-NEXT:    mov x10, v7.d[1]
; CHECK-NOFP16-NEXT:    scvtf s7, x9
; CHECK-NOFP16-NEXT:    mov v19.s[1], v18.s[0]
; CHECK-NOFP16-NEXT:    scvtf s16, x8
; CHECK-NOFP16-NEXT:    mov v0.s[2], v4.s[0]
; CHECK-NOFP16-NEXT:    scvtf s5, x11
; CHECK-NOFP16-NEXT:    mov v3.s[2], v6.s[0]
; CHECK-NOFP16-NEXT:    scvtf s4, x10
; CHECK-NOFP16-NEXT:    mov v2.s[2], v1.s[0]
; CHECK-NOFP16-NEXT:    scvtf s1, x12
; CHECK-NOFP16-NEXT:    mov v0.s[3], v7.s[0]
; CHECK-NOFP16-NEXT:    mov v19.s[2], v5.s[0]
; CHECK-NOFP16-NEXT:    mov v2.s[3], v16.s[0]
; CHECK-NOFP16-NEXT:    mov v3.s[3], v4.s[0]
; CHECK-NOFP16-NEXT:    movi v4.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ushr v5.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    mov v19.s[3], v1.s[0]
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    ushr v6.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    ushr v7.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    add v17.4s, v0.4s, v4.4s
; CHECK-NOFP16-NEXT:    add v18.4s, v2.4s, v4.4s
; CHECK-NOFP16-NEXT:    add v20.4s, v3.4s, v4.4s
; CHECK-NOFP16-NEXT:    ushr v16.4s, v19.4s, #16
; CHECK-NOFP16-NEXT:    and v5.16b, v5.16b, v1.16b
; CHECK-NOFP16-NEXT:    add v4.4s, v19.4s, v4.4s
; CHECK-NOFP16-NEXT:    and v6.16b, v6.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v7.16b, v7.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v1.16b, v16.16b, v1.16b
; CHECK-NOFP16-NEXT:    add v5.4s, v5.4s, v17.4s
; CHECK-NOFP16-NEXT:    fcmeq v16.4s, v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    add v6.4s, v6.4s, v18.4s
; CHECK-NOFP16-NEXT:    fcmeq v17.4s, v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    fcmeq v18.4s, v3.4s, v3.4s
; CHECK-NOFP16-NEXT:    orr v0.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v7.4s, v7.4s, v20.4s
; CHECK-NOFP16-NEXT:    orr v2.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v1.4s, v1.4s, v4.4s
; CHECK-NOFP16-NEXT:    fcmeq v4.4s, v19.4s, v19.4s
; CHECK-NOFP16-NEXT:    orr v3.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    orr v19.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    bit v0.16b, v5.16b, v16.16b
; CHECK-NOFP16-NEXT:    bit v2.16b, v6.16b, v17.16b
; CHECK-NOFP16-NEXT:    bit v3.16b, v7.16b, v18.16b
; CHECK-NOFP16-NEXT:    bif v1.16b, v19.16b, v4.16b
; CHECK-NOFP16-NEXT:    uzp2 v0.8h, v2.8h, v0.8h
; CHECK-NOFP16-NEXT:    uzp2 v1.8h, v1.8h, v3.8h
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v16i64_v16bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    fmov x12, d0
; CHECK-FP16-NEXT:    mov x8, v0.d[1]
; CHECK-FP16-NEXT:    mov x9, v4.d[1]
; CHECK-FP16-NEXT:    mov x10, v2.d[1]
; CHECK-FP16-NEXT:    fmov x11, d2
; CHECK-FP16-NEXT:    mov x13, v1.d[1]
; CHECK-FP16-NEXT:    scvtf s0, x12
; CHECK-FP16-NEXT:    fmov x12, d4
; CHECK-FP16-NEXT:    scvtf s2, x8
; CHECK-FP16-NEXT:    mov x8, v6.d[1]
; CHECK-FP16-NEXT:    scvtf s16, x9
; CHECK-FP16-NEXT:    fmov x9, d1
; CHECK-FP16-NEXT:    scvtf s1, x10
; CHECK-FP16-NEXT:    fmov x10, d5
; CHECK-FP16-NEXT:    scvtf s4, x12
; CHECK-FP16-NEXT:    fmov x12, d6
; CHECK-FP16-NEXT:    scvtf s6, x11
; CHECK-FP16-NEXT:    mov x11, v5.d[1]
; CHECK-FP16-NEXT:    scvtf s5, x9
; CHECK-FP16-NEXT:    mov v0.s[1], v2.s[0]
; CHECK-FP16-NEXT:    scvtf s2, x10
; CHECK-FP16-NEXT:    scvtf s17, x12
; CHECK-FP16-NEXT:    fmov x9, d3
; CHECK-FP16-NEXT:    fmov x12, d7
; CHECK-FP16-NEXT:    mov v4.s[1], v16.s[0]
; CHECK-FP16-NEXT:    scvtf s16, x8
; CHECK-FP16-NEXT:    mov x8, v3.d[1]
; CHECK-FP16-NEXT:    mov x10, v7.d[1]
; CHECK-FP16-NEXT:    mov v6.s[1], v1.s[0]
; CHECK-FP16-NEXT:    scvtf s7, x11
; CHECK-FP16-NEXT:    scvtf s3, x9
; CHECK-FP16-NEXT:    scvtf s1, x12
; CHECK-FP16-NEXT:    mov v0.s[2], v5.s[0]
; CHECK-FP16-NEXT:    scvtf s5, x13
; CHECK-FP16-NEXT:    mov v17.s[1], v16.s[0]
; CHECK-FP16-NEXT:    mov v4.s[2], v2.s[0]
; CHECK-FP16-NEXT:    scvtf s2, x8
; CHECK-FP16-NEXT:    mov v6.s[2], v3.s[0]
; CHECK-FP16-NEXT:    mov v0.s[3], v5.s[0]
; CHECK-FP16-NEXT:    mov v17.s[2], v1.s[0]
; CHECK-FP16-NEXT:    scvtf s1, x10
; CHECK-FP16-NEXT:    mov v4.s[3], v7.s[0]
; CHECK-FP16-NEXT:    mov v6.s[3], v2.s[0]
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    mov v17.s[3], v1.s[0]
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v4.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v6.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v17.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <16 x i64> %a to <16 x bfloat>
  ret <16 x bfloat> %c
}

define <16 x bfloat> @utofp_v16i64_v16bf16(<16 x i64> %a) {
; CHECK-NOFP16-LABEL: utofp_v16i64_v16bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    mov x8, v2.d[1]
; CHECK-NOFP16-NEXT:    fmov x11, d0
; CHECK-NOFP16-NEXT:    mov x9, v0.d[1]
; CHECK-NOFP16-NEXT:    fmov x10, d2
; CHECK-NOFP16-NEXT:    mov x12, v6.d[1]
; CHECK-NOFP16-NEXT:    ucvtf s2, x11
; CHECK-NOFP16-NEXT:    fmov x11, d3
; CHECK-NOFP16-NEXT:    ucvtf s16, x8
; CHECK-NOFP16-NEXT:    fmov x8, d6
; CHECK-NOFP16-NEXT:    ucvtf s0, x10
; CHECK-NOFP16-NEXT:    mov x10, v4.d[1]
; CHECK-NOFP16-NEXT:    ucvtf s17, x9
; CHECK-NOFP16-NEXT:    mov x9, v3.d[1]
; CHECK-NOFP16-NEXT:    ucvtf s6, x12
; CHECK-NOFP16-NEXT:    fmov x12, d4
; CHECK-NOFP16-NEXT:    ucvtf s4, x11
; CHECK-NOFP16-NEXT:    ucvtf s3, x8
; CHECK-NOFP16-NEXT:    fmov x11, d7
; CHECK-NOFP16-NEXT:    mov x8, v1.d[1]
; CHECK-NOFP16-NEXT:    mov v0.s[1], v16.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s18, x10
; CHECK-NOFP16-NEXT:    ucvtf s19, x12
; CHECK-NOFP16-NEXT:    fmov x10, d1
; CHECK-NOFP16-NEXT:    mov v2.s[1], v17.s[0]
; CHECK-NOFP16-NEXT:    mov x12, v5.d[1]
; CHECK-NOFP16-NEXT:    mov v3.s[1], v6.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s6, x11
; CHECK-NOFP16-NEXT:    fmov x11, d5
; CHECK-NOFP16-NEXT:    ucvtf s1, x10
; CHECK-NOFP16-NEXT:    mov x10, v7.d[1]
; CHECK-NOFP16-NEXT:    ucvtf s7, x9
; CHECK-NOFP16-NEXT:    mov v19.s[1], v18.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s16, x8
; CHECK-NOFP16-NEXT:    mov v0.s[2], v4.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s5, x11
; CHECK-NOFP16-NEXT:    mov v3.s[2], v6.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s4, x10
; CHECK-NOFP16-NEXT:    mov v2.s[2], v1.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s1, x12
; CHECK-NOFP16-NEXT:    mov v0.s[3], v7.s[0]
; CHECK-NOFP16-NEXT:    mov v19.s[2], v5.s[0]
; CHECK-NOFP16-NEXT:    mov v2.s[3], v16.s[0]
; CHECK-NOFP16-NEXT:    mov v3.s[3], v4.s[0]
; CHECK-NOFP16-NEXT:    movi v4.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ushr v5.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    mov v19.s[3], v1.s[0]
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    ushr v6.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    ushr v7.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    add v17.4s, v0.4s, v4.4s
; CHECK-NOFP16-NEXT:    add v18.4s, v2.4s, v4.4s
; CHECK-NOFP16-NEXT:    add v20.4s, v3.4s, v4.4s
; CHECK-NOFP16-NEXT:    ushr v16.4s, v19.4s, #16
; CHECK-NOFP16-NEXT:    and v5.16b, v5.16b, v1.16b
; CHECK-NOFP16-NEXT:    add v4.4s, v19.4s, v4.4s
; CHECK-NOFP16-NEXT:    and v6.16b, v6.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v7.16b, v7.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v1.16b, v16.16b, v1.16b
; CHECK-NOFP16-NEXT:    add v5.4s, v5.4s, v17.4s
; CHECK-NOFP16-NEXT:    fcmeq v16.4s, v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    add v6.4s, v6.4s, v18.4s
; CHECK-NOFP16-NEXT:    fcmeq v17.4s, v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    fcmeq v18.4s, v3.4s, v3.4s
; CHECK-NOFP16-NEXT:    orr v0.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v7.4s, v7.4s, v20.4s
; CHECK-NOFP16-NEXT:    orr v2.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v1.4s, v1.4s, v4.4s
; CHECK-NOFP16-NEXT:    fcmeq v4.4s, v19.4s, v19.4s
; CHECK-NOFP16-NEXT:    orr v3.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    orr v19.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    bit v0.16b, v5.16b, v16.16b
; CHECK-NOFP16-NEXT:    bit v2.16b, v6.16b, v17.16b
; CHECK-NOFP16-NEXT:    bit v3.16b, v7.16b, v18.16b
; CHECK-NOFP16-NEXT:    bif v1.16b, v19.16b, v4.16b
; CHECK-NOFP16-NEXT:    uzp2 v0.8h, v2.8h, v0.8h
; CHECK-NOFP16-NEXT:    uzp2 v1.8h, v1.8h, v3.8h
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v16i64_v16bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    fmov x12, d0
; CHECK-FP16-NEXT:    mov x8, v0.d[1]
; CHECK-FP16-NEXT:    mov x9, v4.d[1]
; CHECK-FP16-NEXT:    mov x10, v2.d[1]
; CHECK-FP16-NEXT:    fmov x11, d2
; CHECK-FP16-NEXT:    mov x13, v1.d[1]
; CHECK-FP16-NEXT:    ucvtf s0, x12
; CHECK-FP16-NEXT:    fmov x12, d4
; CHECK-FP16-NEXT:    ucvtf s2, x8
; CHECK-FP16-NEXT:    mov x8, v6.d[1]
; CHECK-FP16-NEXT:    ucvtf s16, x9
; CHECK-FP16-NEXT:    fmov x9, d1
; CHECK-FP16-NEXT:    ucvtf s1, x10
; CHECK-FP16-NEXT:    fmov x10, d5
; CHECK-FP16-NEXT:    ucvtf s4, x12
; CHECK-FP16-NEXT:    fmov x12, d6
; CHECK-FP16-NEXT:    ucvtf s6, x11
; CHECK-FP16-NEXT:    mov x11, v5.d[1]
; CHECK-FP16-NEXT:    ucvtf s5, x9
; CHECK-FP16-NEXT:    mov v0.s[1], v2.s[0]
; CHECK-FP16-NEXT:    ucvtf s2, x10
; CHECK-FP16-NEXT:    ucvtf s17, x12
; CHECK-FP16-NEXT:    fmov x9, d3
; CHECK-FP16-NEXT:    fmov x12, d7
; CHECK-FP16-NEXT:    mov v4.s[1], v16.s[0]
; CHECK-FP16-NEXT:    ucvtf s16, x8
; CHECK-FP16-NEXT:    mov x8, v3.d[1]
; CHECK-FP16-NEXT:    mov x10, v7.d[1]
; CHECK-FP16-NEXT:    mov v6.s[1], v1.s[0]
; CHECK-FP16-NEXT:    ucvtf s7, x11
; CHECK-FP16-NEXT:    ucvtf s3, x9
; CHECK-FP16-NEXT:    ucvtf s1, x12
; CHECK-FP16-NEXT:    mov v0.s[2], v5.s[0]
; CHECK-FP16-NEXT:    ucvtf s5, x13
; CHECK-FP16-NEXT:    mov v17.s[1], v16.s[0]
; CHECK-FP16-NEXT:    mov v4.s[2], v2.s[0]
; CHECK-FP16-NEXT:    ucvtf s2, x8
; CHECK-FP16-NEXT:    mov v6.s[2], v3.s[0]
; CHECK-FP16-NEXT:    mov v0.s[3], v5.s[0]
; CHECK-FP16-NEXT:    mov v17.s[2], v1.s[0]
; CHECK-FP16-NEXT:    ucvtf s1, x10
; CHECK-FP16-NEXT:    mov v4.s[3], v7.s[0]
; CHECK-FP16-NEXT:    mov v6.s[3], v2.s[0]
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    mov v17.s[3], v1.s[0]
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v4.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v6.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v17.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <16 x i64> %a to <16 x bfloat>
  ret <16 x bfloat> %c
}

define <32 x bfloat> @stofp_v32i64_v32bf16(<32 x i64> %a) {
; CHECK-NOFP16-LABEL: stofp_v32i64_v32bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ldp x8, x9, [sp, #32]
; CHECK-NOFP16-NEXT:    mov x13, v2.d[1]
; CHECK-NOFP16-NEXT:    ldp x10, x12, [sp, #96]
; CHECK-NOFP16-NEXT:    fmov x14, d3
; CHECK-NOFP16-NEXT:    movi v17.4s, #1
; CHECK-NOFP16-NEXT:    scvtf s18, x9
; CHECK-NOFP16-NEXT:    scvtf s16, x8
; CHECK-NOFP16-NEXT:    ldp x8, x9, [sp, #48]
; CHECK-NOFP16-NEXT:    scvtf s23, x12
; CHECK-NOFP16-NEXT:    scvtf s20, x10
; CHECK-NOFP16-NEXT:    mov x10, v0.d[1]
; CHECK-NOFP16-NEXT:    scvtf s27, x13
; CHECK-NOFP16-NEXT:    scvtf s21, x8
; CHECK-NOFP16-NEXT:    ldp x8, x11, [sp]
; CHECK-NOFP16-NEXT:    mov v16.s[1], v18.s[0]
; CHECK-NOFP16-NEXT:    scvtf s24, x9
; CHECK-NOFP16-NEXT:    movi v18.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    mov v20.s[1], v23.s[0]
; CHECK-NOFP16-NEXT:    scvtf s22, x11
; CHECK-NOFP16-NEXT:    ldp x11, x12, [sp, #16]
; CHECK-NOFP16-NEXT:    scvtf s19, x8
; CHECK-NOFP16-NEXT:    mov x8, v3.d[1]
; CHECK-NOFP16-NEXT:    mov v16.s[2], v21.s[0]
; CHECK-NOFP16-NEXT:    scvtf s25, x11
; CHECK-NOFP16-NEXT:    ldp x9, x11, [sp, #112]
; CHECK-NOFP16-NEXT:    mov v19.s[1], v22.s[0]
; CHECK-NOFP16-NEXT:    scvtf s22, x12
; CHECK-NOFP16-NEXT:    scvtf s26, x9
; CHECK-NOFP16-NEXT:    ldp x9, x12, [sp, #64]
; CHECK-NOFP16-NEXT:    scvtf s23, x11
; CHECK-NOFP16-NEXT:    mov v16.s[3], v24.s[0]
; CHECK-NOFP16-NEXT:    fmov x11, d2
; CHECK-NOFP16-NEXT:    scvtf s24, x12
; CHECK-NOFP16-NEXT:    scvtf s2, x9
; CHECK-NOFP16-NEXT:    mov x9, v6.d[1]
; CHECK-NOFP16-NEXT:    ldp x12, x13, [sp, #80]
; CHECK-NOFP16-NEXT:    scvtf s21, x11
; CHECK-NOFP16-NEXT:    mov x11, v4.d[1]
; CHECK-NOFP16-NEXT:    mov v19.s[2], v25.s[0]
; CHECK-NOFP16-NEXT:    mov v20.s[2], v26.s[0]
; CHECK-NOFP16-NEXT:    ushr v25.4s, v16.4s, #16
; CHECK-NOFP16-NEXT:    scvtf s26, x14
; CHECK-NOFP16-NEXT:    scvtf s3, x12
; CHECK-NOFP16-NEXT:    mov v2.s[1], v24.s[0]
; CHECK-NOFP16-NEXT:    scvtf s24, x10
; CHECK-NOFP16-NEXT:    fmov x10, d6
; CHECK-NOFP16-NEXT:    fmov x12, d0
; CHECK-NOFP16-NEXT:    scvtf s6, x9
; CHECK-NOFP16-NEXT:    mov v21.s[1], v27.s[0]
; CHECK-NOFP16-NEXT:    scvtf s27, x11
; CHECK-NOFP16-NEXT:    fmov x11, d7
; CHECK-NOFP16-NEXT:    mov v19.s[3], v22.s[0]
; CHECK-NOFP16-NEXT:    mov v20.s[3], v23.s[0]
; CHECK-NOFP16-NEXT:    add v22.4s, v16.4s, v18.4s
; CHECK-NOFP16-NEXT:    mov v2.s[2], v3.s[0]
; CHECK-NOFP16-NEXT:    scvtf s3, x10
; CHECK-NOFP16-NEXT:    fmov x10, d4
; CHECK-NOFP16-NEXT:    scvtf s0, x12
; CHECK-NOFP16-NEXT:    and v23.16b, v25.16b, v17.16b
; CHECK-NOFP16-NEXT:    mov x9, v1.d[1]
; CHECK-NOFP16-NEXT:    fmov x12, d5
; CHECK-NOFP16-NEXT:    mov v21.s[2], v26.s[0]
; CHECK-NOFP16-NEXT:    scvtf s25, x13
; CHECK-NOFP16-NEXT:    scvtf s4, x10
; CHECK-NOFP16-NEXT:    fmov x10, d1
; CHECK-NOFP16-NEXT:    add v26.4s, v20.4s, v18.4s
; CHECK-NOFP16-NEXT:    mov v3.s[1], v6.s[0]
; CHECK-NOFP16-NEXT:    scvtf s6, x11
; CHECK-NOFP16-NEXT:    mov x11, v5.d[1]
; CHECK-NOFP16-NEXT:    scvtf s5, x8
; CHECK-NOFP16-NEXT:    mov v0.s[1], v24.s[0]
; CHECK-NOFP16-NEXT:    add v22.4s, v23.4s, v22.4s
; CHECK-NOFP16-NEXT:    scvtf s1, x10
; CHECK-NOFP16-NEXT:    mov x10, v7.d[1]
; CHECK-NOFP16-NEXT:    scvtf s7, x12
; CHECK-NOFP16-NEXT:    mov v4.s[1], v27.s[0]
; CHECK-NOFP16-NEXT:    ushr v23.4s, v19.4s, #16
; CHECK-NOFP16-NEXT:    mov v2.s[3], v25.s[0]
; CHECK-NOFP16-NEXT:    mov v3.s[2], v6.s[0]
; CHECK-NOFP16-NEXT:    add v25.4s, v19.4s, v18.4s
; CHECK-NOFP16-NEXT:    ushr v24.4s, v20.4s, #16
; CHECK-NOFP16-NEXT:    mov v21.s[3], v5.s[0]
; CHECK-NOFP16-NEXT:    scvtf s5, x11
; CHECK-NOFP16-NEXT:    fcmeq v29.4s, v20.4s, v20.4s
; CHECK-NOFP16-NEXT:    scvtf s6, x10
; CHECK-NOFP16-NEXT:    and v23.16b, v23.16b, v17.16b
; CHECK-NOFP16-NEXT:    mov v0.s[2], v1.s[0]
; CHECK-NOFP16-NEXT:    scvtf s1, x9
; CHECK-NOFP16-NEXT:    mov v4.s[2], v7.s[0]
; CHECK-NOFP16-NEXT:    and v24.16b, v24.16b, v17.16b
; CHECK-NOFP16-NEXT:    fcmeq v7.4s, v16.4s, v16.4s
; CHECK-NOFP16-NEXT:    orr v16.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    fcmeq v31.4s, v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    add v27.4s, v21.4s, v18.4s
; CHECK-NOFP16-NEXT:    orr v20.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    mov v3.s[3], v6.s[0]
; CHECK-NOFP16-NEXT:    add v6.4s, v23.4s, v25.4s
; CHECK-NOFP16-NEXT:    ushr v23.4s, v21.4s, #16
; CHECK-NOFP16-NEXT:    mov v0.s[3], v1.s[0]
; CHECK-NOFP16-NEXT:    mov v4.s[3], v5.s[0]
; CHECK-NOFP16-NEXT:    ushr v1.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    add v24.4s, v24.4s, v26.4s
; CHECK-NOFP16-NEXT:    add v25.4s, v2.4s, v18.4s
; CHECK-NOFP16-NEXT:    fcmeq v5.4s, v19.4s, v19.4s
; CHECK-NOFP16-NEXT:    and v23.16b, v23.16b, v17.16b
; CHECK-NOFP16-NEXT:    orr v19.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    orr v2.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    ushr v28.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v1.16b, v17.16b
; CHECK-NOFP16-NEXT:    bsl v7.16b, v22.16b, v16.16b
; CHECK-NOFP16-NEXT:    ushr v26.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    ushr v30.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    add v23.4s, v23.4s, v27.4s
; CHECK-NOFP16-NEXT:    bsl v5.16b, v6.16b, v19.16b
; CHECK-NOFP16-NEXT:    mov v6.16b, v29.16b
; CHECK-NOFP16-NEXT:    and v27.16b, v28.16b, v17.16b
; CHECK-NOFP16-NEXT:    add v28.4s, v3.4s, v18.4s
; CHECK-NOFP16-NEXT:    add v1.4s, v1.4s, v25.4s
; CHECK-NOFP16-NEXT:    and v25.16b, v26.16b, v17.16b
; CHECK-NOFP16-NEXT:    add v26.4s, v0.4s, v18.4s
; CHECK-NOFP16-NEXT:    and v17.16b, v30.16b, v17.16b
; CHECK-NOFP16-NEXT:    add v18.4s, v4.4s, v18.4s
; CHECK-NOFP16-NEXT:    fcmeq v30.4s, v21.4s, v21.4s
; CHECK-NOFP16-NEXT:    orr v21.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v27.4s, v27.4s, v28.4s
; CHECK-NOFP16-NEXT:    fcmeq v28.4s, v3.4s, v3.4s
; CHECK-NOFP16-NEXT:    orr v3.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v25.4s, v25.4s, v26.4s
; CHECK-NOFP16-NEXT:    fcmeq v26.4s, v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    orr v0.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v17.4s, v17.4s, v18.4s
; CHECK-NOFP16-NEXT:    fcmeq v18.4s, v4.4s, v4.4s
; CHECK-NOFP16-NEXT:    orr v4.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    mov v16.16b, v30.16b
; CHECK-NOFP16-NEXT:    bsl v6.16b, v24.16b, v20.16b
; CHECK-NOFP16-NEXT:    bif v1.16b, v2.16b, v31.16b
; CHECK-NOFP16-NEXT:    mov v19.16b, v28.16b
; CHECK-NOFP16-NEXT:    uzp2 v2.8h, v5.8h, v7.8h
; CHECK-NOFP16-NEXT:    bit v0.16b, v25.16b, v26.16b
; CHECK-NOFP16-NEXT:    bsl v16.16b, v23.16b, v21.16b
; CHECK-NOFP16-NEXT:    bit v4.16b, v17.16b, v18.16b
; CHECK-NOFP16-NEXT:    bsl v19.16b, v27.16b, v3.16b
; CHECK-NOFP16-NEXT:    uzp2 v3.8h, v1.8h, v6.8h
; CHECK-NOFP16-NEXT:    uzp2 v0.8h, v0.8h, v16.8h
; CHECK-NOFP16-NEXT:    uzp2 v1.8h, v4.8h, v19.8h
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v32i64_v32bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ldp x8, x9, [sp, #32]
; CHECK-FP16-NEXT:    mov x14, v2.d[1]
; CHECK-FP16-NEXT:    ldp x10, x11, [sp]
; CHECK-FP16-NEXT:    ldp x12, x13, [sp, #96]
; CHECK-FP16-NEXT:    scvtf s18, x9
; CHECK-FP16-NEXT:    scvtf s16, x8
; CHECK-FP16-NEXT:    ldp x8, x9, [sp, #64]
; CHECK-FP16-NEXT:    scvtf s17, x10
; CHECK-FP16-NEXT:    scvtf s19, x11
; CHECK-FP16-NEXT:    scvtf s21, x13
; CHECK-FP16-NEXT:    mov x13, v4.d[1]
; CHECK-FP16-NEXT:    scvtf s20, x9
; CHECK-FP16-NEXT:    ldp x10, x9, [sp, #16]
; CHECK-FP16-NEXT:    mov v16.s[1], v18.s[0]
; CHECK-FP16-NEXT:    scvtf s18, x8
; CHECK-FP16-NEXT:    mov v17.s[1], v19.s[0]
; CHECK-FP16-NEXT:    scvtf s19, x12
; CHECK-FP16-NEXT:    mov x12, v0.d[1]
; CHECK-FP16-NEXT:    scvtf s22, x10
; CHECK-FP16-NEXT:    ldp x11, x10, [sp, #80]
; CHECK-FP16-NEXT:    ldp x15, x8, [sp, #48]
; CHECK-FP16-NEXT:    mov v18.s[1], v20.s[0]
; CHECK-FP16-NEXT:    scvtf s23, x11
; CHECK-FP16-NEXT:    fmov x11, d2
; CHECK-FP16-NEXT:    mov v19.s[1], v21.s[0]
; CHECK-FP16-NEXT:    mov v17.s[2], v22.s[0]
; CHECK-FP16-NEXT:    scvtf s22, x14
; CHECK-FP16-NEXT:    fmov x14, d1
; CHECK-FP16-NEXT:    scvtf s2, x15
; CHECK-FP16-NEXT:    scvtf s20, x11
; CHECK-FP16-NEXT:    fmov x11, d0
; CHECK-FP16-NEXT:    scvtf s0, x12
; CHECK-FP16-NEXT:    fmov x12, d4
; CHECK-FP16-NEXT:    mov v18.s[2], v23.s[0]
; CHECK-FP16-NEXT:    scvtf s23, x9
; CHECK-FP16-NEXT:    mov x9, v6.d[1]
; CHECK-FP16-NEXT:    scvtf s21, x11
; CHECK-FP16-NEXT:    mov x11, v1.d[1]
; CHECK-FP16-NEXT:    scvtf s1, x13
; CHECK-FP16-NEXT:    scvtf s4, x12
; CHECK-FP16-NEXT:    fmov x12, d6
; CHECK-FP16-NEXT:    mov v20.s[1], v22.s[0]
; CHECK-FP16-NEXT:    mov x13, v5.d[1]
; CHECK-FP16-NEXT:    mov v16.s[2], v2.s[0]
; CHECK-FP16-NEXT:    mov v17.s[3], v23.s[0]
; CHECK-FP16-NEXT:    scvtf s6, x9
; CHECK-FP16-NEXT:    scvtf s2, x8
; CHECK-FP16-NEXT:    mov v21.s[1], v0.s[0]
; CHECK-FP16-NEXT:    scvtf s0, x14
; CHECK-FP16-NEXT:    scvtf s22, x12
; CHECK-FP16-NEXT:    fmov x14, d5
; CHECK-FP16-NEXT:    mov v4.s[1], v1.s[0]
; CHECK-FP16-NEXT:    ldp x9, x12, [sp, #112]
; CHECK-FP16-NEXT:    mov v16.s[3], v2.s[0]
; CHECK-FP16-NEXT:    bfcvtn v2.4h, v17.4s
; CHECK-FP16-NEXT:    scvtf s1, x14
; CHECK-FP16-NEXT:    fmov x14, d3
; CHECK-FP16-NEXT:    mov v22.s[1], v6.s[0]
; CHECK-FP16-NEXT:    scvtf s6, x11
; CHECK-FP16-NEXT:    fmov x11, d7
; CHECK-FP16-NEXT:    scvtf s5, x9
; CHECK-FP16-NEXT:    mov x9, v3.d[1]
; CHECK-FP16-NEXT:    scvtf s3, x10
; CHECK-FP16-NEXT:    mov v21.s[2], v0.s[0]
; CHECK-FP16-NEXT:    scvtf s0, x14
; CHECK-FP16-NEXT:    mov x10, v7.d[1]
; CHECK-FP16-NEXT:    bfcvtn2 v2.8h, v16.4s
; CHECK-FP16-NEXT:    scvtf s7, x11
; CHECK-FP16-NEXT:    mov v4.s[2], v1.s[0]
; CHECK-FP16-NEXT:    scvtf s1, x13
; CHECK-FP16-NEXT:    mov v19.s[2], v5.s[0]
; CHECK-FP16-NEXT:    mov v18.s[3], v3.s[0]
; CHECK-FP16-NEXT:    mov v21.s[3], v6.s[0]
; CHECK-FP16-NEXT:    scvtf s3, x9
; CHECK-FP16-NEXT:    mov v20.s[2], v0.s[0]
; CHECK-FP16-NEXT:    scvtf s0, x12
; CHECK-FP16-NEXT:    mov v22.s[2], v7.s[0]
; CHECK-FP16-NEXT:    mov v4.s[3], v1.s[0]
; CHECK-FP16-NEXT:    scvtf s1, x10
; CHECK-FP16-NEXT:    mov v19.s[3], v0.s[0]
; CHECK-FP16-NEXT:    mov v20.s[3], v3.s[0]
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v21.4s
; CHECK-FP16-NEXT:    bfcvtn v3.4h, v18.4s
; CHECK-FP16-NEXT:    mov v22.s[3], v1.s[0]
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v4.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v20.4s
; CHECK-FP16-NEXT:    bfcvtn2 v3.8h, v19.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v22.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <32 x i64> %a to <32 x bfloat>
  ret <32 x bfloat> %c
}

define <32 x bfloat> @utofp_v32i64_v32bf16(<32 x i64> %a) {
; CHECK-NOFP16-LABEL: utofp_v32i64_v32bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ldp x8, x9, [sp, #32]
; CHECK-NOFP16-NEXT:    mov x13, v2.d[1]
; CHECK-NOFP16-NEXT:    ldp x10, x12, [sp, #96]
; CHECK-NOFP16-NEXT:    fmov x14, d3
; CHECK-NOFP16-NEXT:    movi v17.4s, #1
; CHECK-NOFP16-NEXT:    ucvtf s18, x9
; CHECK-NOFP16-NEXT:    ucvtf s16, x8
; CHECK-NOFP16-NEXT:    ldp x8, x9, [sp, #48]
; CHECK-NOFP16-NEXT:    ucvtf s23, x12
; CHECK-NOFP16-NEXT:    ucvtf s20, x10
; CHECK-NOFP16-NEXT:    mov x10, v0.d[1]
; CHECK-NOFP16-NEXT:    ucvtf s27, x13
; CHECK-NOFP16-NEXT:    ucvtf s21, x8
; CHECK-NOFP16-NEXT:    ldp x8, x11, [sp]
; CHECK-NOFP16-NEXT:    mov v16.s[1], v18.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s24, x9
; CHECK-NOFP16-NEXT:    movi v18.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    mov v20.s[1], v23.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s22, x11
; CHECK-NOFP16-NEXT:    ldp x11, x12, [sp, #16]
; CHECK-NOFP16-NEXT:    ucvtf s19, x8
; CHECK-NOFP16-NEXT:    mov x8, v3.d[1]
; CHECK-NOFP16-NEXT:    mov v16.s[2], v21.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s25, x11
; CHECK-NOFP16-NEXT:    ldp x9, x11, [sp, #112]
; CHECK-NOFP16-NEXT:    mov v19.s[1], v22.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s22, x12
; CHECK-NOFP16-NEXT:    ucvtf s26, x9
; CHECK-NOFP16-NEXT:    ldp x9, x12, [sp, #64]
; CHECK-NOFP16-NEXT:    ucvtf s23, x11
; CHECK-NOFP16-NEXT:    mov v16.s[3], v24.s[0]
; CHECK-NOFP16-NEXT:    fmov x11, d2
; CHECK-NOFP16-NEXT:    ucvtf s24, x12
; CHECK-NOFP16-NEXT:    ucvtf s2, x9
; CHECK-NOFP16-NEXT:    mov x9, v6.d[1]
; CHECK-NOFP16-NEXT:    ldp x12, x13, [sp, #80]
; CHECK-NOFP16-NEXT:    ucvtf s21, x11
; CHECK-NOFP16-NEXT:    mov x11, v4.d[1]
; CHECK-NOFP16-NEXT:    mov v19.s[2], v25.s[0]
; CHECK-NOFP16-NEXT:    mov v20.s[2], v26.s[0]
; CHECK-NOFP16-NEXT:    ushr v25.4s, v16.4s, #16
; CHECK-NOFP16-NEXT:    ucvtf s26, x14
; CHECK-NOFP16-NEXT:    ucvtf s3, x12
; CHECK-NOFP16-NEXT:    mov v2.s[1], v24.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s24, x10
; CHECK-NOFP16-NEXT:    fmov x10, d6
; CHECK-NOFP16-NEXT:    fmov x12, d0
; CHECK-NOFP16-NEXT:    ucvtf s6, x9
; CHECK-NOFP16-NEXT:    mov v21.s[1], v27.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s27, x11
; CHECK-NOFP16-NEXT:    fmov x11, d7
; CHECK-NOFP16-NEXT:    mov v19.s[3], v22.s[0]
; CHECK-NOFP16-NEXT:    mov v20.s[3], v23.s[0]
; CHECK-NOFP16-NEXT:    add v22.4s, v16.4s, v18.4s
; CHECK-NOFP16-NEXT:    mov v2.s[2], v3.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s3, x10
; CHECK-NOFP16-NEXT:    fmov x10, d4
; CHECK-NOFP16-NEXT:    ucvtf s0, x12
; CHECK-NOFP16-NEXT:    and v23.16b, v25.16b, v17.16b
; CHECK-NOFP16-NEXT:    mov x9, v1.d[1]
; CHECK-NOFP16-NEXT:    fmov x12, d5
; CHECK-NOFP16-NEXT:    mov v21.s[2], v26.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s25, x13
; CHECK-NOFP16-NEXT:    ucvtf s4, x10
; CHECK-NOFP16-NEXT:    fmov x10, d1
; CHECK-NOFP16-NEXT:    add v26.4s, v20.4s, v18.4s
; CHECK-NOFP16-NEXT:    mov v3.s[1], v6.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s6, x11
; CHECK-NOFP16-NEXT:    mov x11, v5.d[1]
; CHECK-NOFP16-NEXT:    ucvtf s5, x8
; CHECK-NOFP16-NEXT:    mov v0.s[1], v24.s[0]
; CHECK-NOFP16-NEXT:    add v22.4s, v23.4s, v22.4s
; CHECK-NOFP16-NEXT:    ucvtf s1, x10
; CHECK-NOFP16-NEXT:    mov x10, v7.d[1]
; CHECK-NOFP16-NEXT:    ucvtf s7, x12
; CHECK-NOFP16-NEXT:    mov v4.s[1], v27.s[0]
; CHECK-NOFP16-NEXT:    ushr v23.4s, v19.4s, #16
; CHECK-NOFP16-NEXT:    mov v2.s[3], v25.s[0]
; CHECK-NOFP16-NEXT:    mov v3.s[2], v6.s[0]
; CHECK-NOFP16-NEXT:    add v25.4s, v19.4s, v18.4s
; CHECK-NOFP16-NEXT:    ushr v24.4s, v20.4s, #16
; CHECK-NOFP16-NEXT:    mov v21.s[3], v5.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s5, x11
; CHECK-NOFP16-NEXT:    fcmeq v29.4s, v20.4s, v20.4s
; CHECK-NOFP16-NEXT:    ucvtf s6, x10
; CHECK-NOFP16-NEXT:    and v23.16b, v23.16b, v17.16b
; CHECK-NOFP16-NEXT:    mov v0.s[2], v1.s[0]
; CHECK-NOFP16-NEXT:    ucvtf s1, x9
; CHECK-NOFP16-NEXT:    mov v4.s[2], v7.s[0]
; CHECK-NOFP16-NEXT:    and v24.16b, v24.16b, v17.16b
; CHECK-NOFP16-NEXT:    fcmeq v7.4s, v16.4s, v16.4s
; CHECK-NOFP16-NEXT:    orr v16.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    fcmeq v31.4s, v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    add v27.4s, v21.4s, v18.4s
; CHECK-NOFP16-NEXT:    orr v20.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    mov v3.s[3], v6.s[0]
; CHECK-NOFP16-NEXT:    add v6.4s, v23.4s, v25.4s
; CHECK-NOFP16-NEXT:    ushr v23.4s, v21.4s, #16
; CHECK-NOFP16-NEXT:    mov v0.s[3], v1.s[0]
; CHECK-NOFP16-NEXT:    mov v4.s[3], v5.s[0]
; CHECK-NOFP16-NEXT:    ushr v1.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    add v24.4s, v24.4s, v26.4s
; CHECK-NOFP16-NEXT:    add v25.4s, v2.4s, v18.4s
; CHECK-NOFP16-NEXT:    fcmeq v5.4s, v19.4s, v19.4s
; CHECK-NOFP16-NEXT:    and v23.16b, v23.16b, v17.16b
; CHECK-NOFP16-NEXT:    orr v19.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    orr v2.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    ushr v28.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v1.16b, v17.16b
; CHECK-NOFP16-NEXT:    bsl v7.16b, v22.16b, v16.16b
; CHECK-NOFP16-NEXT:    ushr v26.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    ushr v30.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    add v23.4s, v23.4s, v27.4s
; CHECK-NOFP16-NEXT:    bsl v5.16b, v6.16b, v19.16b
; CHECK-NOFP16-NEXT:    mov v6.16b, v29.16b
; CHECK-NOFP16-NEXT:    and v27.16b, v28.16b, v17.16b
; CHECK-NOFP16-NEXT:    add v28.4s, v3.4s, v18.4s
; CHECK-NOFP16-NEXT:    add v1.4s, v1.4s, v25.4s
; CHECK-NOFP16-NEXT:    and v25.16b, v26.16b, v17.16b
; CHECK-NOFP16-NEXT:    add v26.4s, v0.4s, v18.4s
; CHECK-NOFP16-NEXT:    and v17.16b, v30.16b, v17.16b
; CHECK-NOFP16-NEXT:    add v18.4s, v4.4s, v18.4s
; CHECK-NOFP16-NEXT:    fcmeq v30.4s, v21.4s, v21.4s
; CHECK-NOFP16-NEXT:    orr v21.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v27.4s, v27.4s, v28.4s
; CHECK-NOFP16-NEXT:    fcmeq v28.4s, v3.4s, v3.4s
; CHECK-NOFP16-NEXT:    orr v3.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v25.4s, v25.4s, v26.4s
; CHECK-NOFP16-NEXT:    fcmeq v26.4s, v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    orr v0.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    add v17.4s, v17.4s, v18.4s
; CHECK-NOFP16-NEXT:    fcmeq v18.4s, v4.4s, v4.4s
; CHECK-NOFP16-NEXT:    orr v4.4s, #64, lsl #16
; CHECK-NOFP16-NEXT:    mov v16.16b, v30.16b
; CHECK-NOFP16-NEXT:    bsl v6.16b, v24.16b, v20.16b
; CHECK-NOFP16-NEXT:    bif v1.16b, v2.16b, v31.16b
; CHECK-NOFP16-NEXT:    mov v19.16b, v28.16b
; CHECK-NOFP16-NEXT:    uzp2 v2.8h, v5.8h, v7.8h
; CHECK-NOFP16-NEXT:    bit v0.16b, v25.16b, v26.16b
; CHECK-NOFP16-NEXT:    bsl v16.16b, v23.16b, v21.16b
; CHECK-NOFP16-NEXT:    bit v4.16b, v17.16b, v18.16b
; CHECK-NOFP16-NEXT:    bsl v19.16b, v27.16b, v3.16b
; CHECK-NOFP16-NEXT:    uzp2 v3.8h, v1.8h, v6.8h
; CHECK-NOFP16-NEXT:    uzp2 v0.8h, v0.8h, v16.8h
; CHECK-NOFP16-NEXT:    uzp2 v1.8h, v4.8h, v19.8h
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v32i64_v32bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ldp x8, x9, [sp, #32]
; CHECK-FP16-NEXT:    mov x14, v2.d[1]
; CHECK-FP16-NEXT:    ldp x10, x11, [sp]
; CHECK-FP16-NEXT:    ldp x12, x13, [sp, #96]
; CHECK-FP16-NEXT:    ucvtf s18, x9
; CHECK-FP16-NEXT:    ucvtf s16, x8
; CHECK-FP16-NEXT:    ldp x8, x9, [sp, #64]
; CHECK-FP16-NEXT:    ucvtf s17, x10
; CHECK-FP16-NEXT:    ucvtf s19, x11
; CHECK-FP16-NEXT:    ucvtf s21, x13
; CHECK-FP16-NEXT:    mov x13, v4.d[1]
; CHECK-FP16-NEXT:    ucvtf s20, x9
; CHECK-FP16-NEXT:    ldp x10, x9, [sp, #16]
; CHECK-FP16-NEXT:    mov v16.s[1], v18.s[0]
; CHECK-FP16-NEXT:    ucvtf s18, x8
; CHECK-FP16-NEXT:    mov v17.s[1], v19.s[0]
; CHECK-FP16-NEXT:    ucvtf s19, x12
; CHECK-FP16-NEXT:    mov x12, v0.d[1]
; CHECK-FP16-NEXT:    ucvtf s22, x10
; CHECK-FP16-NEXT:    ldp x11, x10, [sp, #80]
; CHECK-FP16-NEXT:    ldp x15, x8, [sp, #48]
; CHECK-FP16-NEXT:    mov v18.s[1], v20.s[0]
; CHECK-FP16-NEXT:    ucvtf s23, x11
; CHECK-FP16-NEXT:    fmov x11, d2
; CHECK-FP16-NEXT:    mov v19.s[1], v21.s[0]
; CHECK-FP16-NEXT:    mov v17.s[2], v22.s[0]
; CHECK-FP16-NEXT:    ucvtf s22, x14
; CHECK-FP16-NEXT:    fmov x14, d1
; CHECK-FP16-NEXT:    ucvtf s2, x15
; CHECK-FP16-NEXT:    ucvtf s20, x11
; CHECK-FP16-NEXT:    fmov x11, d0
; CHECK-FP16-NEXT:    ucvtf s0, x12
; CHECK-FP16-NEXT:    fmov x12, d4
; CHECK-FP16-NEXT:    mov v18.s[2], v23.s[0]
; CHECK-FP16-NEXT:    ucvtf s23, x9
; CHECK-FP16-NEXT:    mov x9, v6.d[1]
; CHECK-FP16-NEXT:    ucvtf s21, x11
; CHECK-FP16-NEXT:    mov x11, v1.d[1]
; CHECK-FP16-NEXT:    ucvtf s1, x13
; CHECK-FP16-NEXT:    ucvtf s4, x12
; CHECK-FP16-NEXT:    fmov x12, d6
; CHECK-FP16-NEXT:    mov v20.s[1], v22.s[0]
; CHECK-FP16-NEXT:    mov x13, v5.d[1]
; CHECK-FP16-NEXT:    mov v16.s[2], v2.s[0]
; CHECK-FP16-NEXT:    mov v17.s[3], v23.s[0]
; CHECK-FP16-NEXT:    ucvtf s6, x9
; CHECK-FP16-NEXT:    ucvtf s2, x8
; CHECK-FP16-NEXT:    mov v21.s[1], v0.s[0]
; CHECK-FP16-NEXT:    ucvtf s0, x14
; CHECK-FP16-NEXT:    ucvtf s22, x12
; CHECK-FP16-NEXT:    fmov x14, d5
; CHECK-FP16-NEXT:    mov v4.s[1], v1.s[0]
; CHECK-FP16-NEXT:    ldp x9, x12, [sp, #112]
; CHECK-FP16-NEXT:    mov v16.s[3], v2.s[0]
; CHECK-FP16-NEXT:    bfcvtn v2.4h, v17.4s
; CHECK-FP16-NEXT:    ucvtf s1, x14
; CHECK-FP16-NEXT:    fmov x14, d3
; CHECK-FP16-NEXT:    mov v22.s[1], v6.s[0]
; CHECK-FP16-NEXT:    ucvtf s6, x11
; CHECK-FP16-NEXT:    fmov x11, d7
; CHECK-FP16-NEXT:    ucvtf s5, x9
; CHECK-FP16-NEXT:    mov x9, v3.d[1]
; CHECK-FP16-NEXT:    ucvtf s3, x10
; CHECK-FP16-NEXT:    mov v21.s[2], v0.s[0]
; CHECK-FP16-NEXT:    ucvtf s0, x14
; CHECK-FP16-NEXT:    mov x10, v7.d[1]
; CHECK-FP16-NEXT:    bfcvtn2 v2.8h, v16.4s
; CHECK-FP16-NEXT:    ucvtf s7, x11
; CHECK-FP16-NEXT:    mov v4.s[2], v1.s[0]
; CHECK-FP16-NEXT:    ucvtf s1, x13
; CHECK-FP16-NEXT:    mov v19.s[2], v5.s[0]
; CHECK-FP16-NEXT:    mov v18.s[3], v3.s[0]
; CHECK-FP16-NEXT:    mov v21.s[3], v6.s[0]
; CHECK-FP16-NEXT:    ucvtf s3, x9
; CHECK-FP16-NEXT:    mov v20.s[2], v0.s[0]
; CHECK-FP16-NEXT:    ucvtf s0, x12
; CHECK-FP16-NEXT:    mov v22.s[2], v7.s[0]
; CHECK-FP16-NEXT:    mov v4.s[3], v1.s[0]
; CHECK-FP16-NEXT:    ucvtf s1, x10
; CHECK-FP16-NEXT:    mov v19.s[3], v0.s[0]
; CHECK-FP16-NEXT:    mov v20.s[3], v3.s[0]
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v21.4s
; CHECK-FP16-NEXT:    bfcvtn v3.4h, v18.4s
; CHECK-FP16-NEXT:    mov v22.s[3], v1.s[0]
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v4.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v20.4s
; CHECK-FP16-NEXT:    bfcvtn2 v3.8h, v19.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v22.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <32 x i64> %a to <32 x bfloat>
  ret <32 x bfloat> %c
}

define <2 x bfloat> @stofp_v2i32_v2bf16(<2 x i32> %a) {
; CHECK-NOFP16-LABEL: stofp_v2i32_v2bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v2i32_v2bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-FP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <2 x i32> %a to <2 x bfloat>
  ret <2 x bfloat> %c
}

define <2 x bfloat> @utofp_v2i32_v2bf16(<2 x i32> %a) {
; CHECK-NOFP16-LABEL: utofp_v2i32_v2bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v2i32_v2bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-FP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <2 x i32> %a to <2 x bfloat>
  ret <2 x bfloat> %c
}

define <3 x bfloat> @stofp_v3i32_v3bf16(<3 x i32> %a) {
; CHECK-NOFP16-LABEL: stofp_v3i32_v3bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v3i32_v3bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <3 x i32> %a to <3 x bfloat>
  ret <3 x bfloat> %c
}

define <3 x bfloat> @utofp_v3i32_v3bf16(<3 x i32> %a) {
; CHECK-NOFP16-LABEL: utofp_v3i32_v3bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v3i32_v3bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <3 x i32> %a to <3 x bfloat>
  ret <3 x bfloat> %c
}

define <4 x bfloat> @stofp_v4i32_v4bf16(<4 x i32> %a) {
; CHECK-NOFP16-LABEL: stofp_v4i32_v4bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v4i32_v4bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <4 x i32> %a to <4 x bfloat>
  ret <4 x bfloat> %c
}

define <4 x bfloat> @utofp_v4i32_v4bf16(<4 x i32> %a) {
; CHECK-NOFP16-LABEL: utofp_v4i32_v4bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v4i32_v4bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <4 x i32> %a to <4 x bfloat>
  ret <4 x bfloat> %c
}

define <8 x bfloat> @stofp_v8i32_v8bf16(<8 x i32> %a) {
; CHECK-NOFP16-LABEL: stofp_v8i32_v8bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    movi v2.4s, #1
; CHECK-NOFP16-NEXT:    scvtf v1.4s, v1.4s
; CHECK-NOFP16-NEXT:    movi v5.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ushr v3.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    ushr v4.4s, v1.4s, #16
; CHECK-NOFP16-NEXT:    and v3.16b, v3.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v2.16b, v4.16b, v2.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-NOFP16-NEXT:    add v1.4s, v2.4s, v1.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v5.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v1.4s, v5.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v8i32_v8bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    scvtf v1.4s, v1.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v1.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <8 x i32> %a to <8 x bfloat>
  ret <8 x bfloat> %c
}

define <8 x bfloat> @utofp_v8i32_v8bf16(<8 x i32> %a) {
; CHECK-NOFP16-LABEL: utofp_v8i32_v8bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    movi v2.4s, #1
; CHECK-NOFP16-NEXT:    ucvtf v1.4s, v1.4s
; CHECK-NOFP16-NEXT:    movi v5.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ushr v3.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    ushr v4.4s, v1.4s, #16
; CHECK-NOFP16-NEXT:    and v3.16b, v3.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v2.16b, v4.16b, v2.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-NOFP16-NEXT:    add v1.4s, v2.4s, v1.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v5.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v1.4s, v5.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v8i32_v8bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    ucvtf v1.4s, v1.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v1.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <8 x i32> %a to <8 x bfloat>
  ret <8 x bfloat> %c
}

define <16 x bfloat> @stofp_v16i32_v16bf16(<16 x i32> %a) {
; CHECK-NOFP16-LABEL: stofp_v16i32_v16bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    scvtf v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    scvtf v4.4s, v1.4s
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    scvtf v3.4s, v3.4s
; CHECK-NOFP16-NEXT:    movi v17.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ushr v5.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    ushr v6.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    ushr v7.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    ushr v16.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    and v5.16b, v5.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v6.16b, v6.16b, v1.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v5.4s, v0.4s
; CHECK-NOFP16-NEXT:    add v2.4s, v6.4s, v2.4s
; CHECK-NOFP16-NEXT:    and v5.16b, v7.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v6.16b, v16.16b, v1.16b
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v17.4s
; CHECK-NOFP16-NEXT:    addhn v1.4h, v2.4s, v17.4s
; CHECK-NOFP16-NEXT:    add v2.4s, v5.4s, v4.4s
; CHECK-NOFP16-NEXT:    add v3.4s, v6.4s, v3.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v2.4s, v17.4s
; CHECK-NOFP16-NEXT:    addhn2 v1.8h, v3.4s, v17.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v16i32_v16bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    scvtf v2.4s, v2.4s
; CHECK-FP16-NEXT:    scvtf v4.4s, v1.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v2.4s
; CHECK-FP16-NEXT:    scvtf v2.4s, v3.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v4.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v2.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <16 x i32> %a to <16 x bfloat>
  ret <16 x bfloat> %c
}

define <16 x bfloat> @utofp_v16i32_v16bf16(<16 x i32> %a) {
; CHECK-NOFP16-LABEL: utofp_v16i32_v16bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ucvtf v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    ucvtf v4.4s, v1.4s
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    ucvtf v3.4s, v3.4s
; CHECK-NOFP16-NEXT:    movi v17.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ushr v5.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    ushr v6.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    ushr v7.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    ushr v16.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    and v5.16b, v5.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v6.16b, v6.16b, v1.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v5.4s, v0.4s
; CHECK-NOFP16-NEXT:    add v2.4s, v6.4s, v2.4s
; CHECK-NOFP16-NEXT:    and v5.16b, v7.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v6.16b, v16.16b, v1.16b
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v17.4s
; CHECK-NOFP16-NEXT:    addhn v1.4h, v2.4s, v17.4s
; CHECK-NOFP16-NEXT:    add v2.4s, v5.4s, v4.4s
; CHECK-NOFP16-NEXT:    add v3.4s, v6.4s, v3.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v2.4s, v17.4s
; CHECK-NOFP16-NEXT:    addhn2 v1.8h, v3.4s, v17.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v16i32_v16bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    ucvtf v2.4s, v2.4s
; CHECK-FP16-NEXT:    ucvtf v4.4s, v1.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v2.4s
; CHECK-FP16-NEXT:    ucvtf v2.4s, v3.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v4.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v2.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <16 x i32> %a to <16 x bfloat>
  ret <16 x bfloat> %c
}

define <32 x bfloat> @stofp_v32i32_v32bf16(<32 x i32> %a) {
; CHECK-NOFP16-LABEL: stofp_v32i32_v32bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    scvtf v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    scvtf v4.4s, v4.4s
; CHECK-NOFP16-NEXT:    scvtf v6.4s, v6.4s
; CHECK-NOFP16-NEXT:    movi v16.4s, #1
; CHECK-NOFP16-NEXT:    scvtf v1.4s, v1.4s
; CHECK-NOFP16-NEXT:    scvtf v17.4s, v3.4s
; CHECK-NOFP16-NEXT:    scvtf v5.4s, v5.4s
; CHECK-NOFP16-NEXT:    scvtf v7.4s, v7.4s
; CHECK-NOFP16-NEXT:    movi v21.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ushr v3.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    ushr v18.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    ushr v19.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    ushr v20.4s, v6.4s, #16
; CHECK-NOFP16-NEXT:    ushr v22.4s, v1.4s, #16
; CHECK-NOFP16-NEXT:    ushr v23.4s, v17.4s, #16
; CHECK-NOFP16-NEXT:    ushr v24.4s, v5.4s, #16
; CHECK-NOFP16-NEXT:    ushr v25.4s, v7.4s, #16
; CHECK-NOFP16-NEXT:    and v3.16b, v3.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v18.16b, v18.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v19.16b, v19.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v20.16b, v20.16b, v16.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-NOFP16-NEXT:    and v3.16b, v22.16b, v16.16b
; CHECK-NOFP16-NEXT:    add v2.4s, v18.4s, v2.4s
; CHECK-NOFP16-NEXT:    add v4.4s, v19.4s, v4.4s
; CHECK-NOFP16-NEXT:    add v6.4s, v20.4s, v6.4s
; CHECK-NOFP16-NEXT:    and v18.16b, v23.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v19.16b, v24.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v16.16b, v25.16b, v16.16b
; CHECK-NOFP16-NEXT:    add v20.4s, v3.4s, v1.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn v1.4h, v2.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn v2.4h, v4.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn v3.4h, v6.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v4.4s, v18.4s, v17.4s
; CHECK-NOFP16-NEXT:    add v5.4s, v19.4s, v5.4s
; CHECK-NOFP16-NEXT:    add v6.4s, v16.4s, v7.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v20.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn2 v1.8h, v4.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn2 v2.8h, v5.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn2 v3.8h, v6.4s, v21.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v32i32_v32bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    scvtf v2.4s, v2.4s
; CHECK-FP16-NEXT:    scvtf v4.4s, v4.4s
; CHECK-FP16-NEXT:    scvtf v6.4s, v6.4s
; CHECK-FP16-NEXT:    scvtf v16.4s, v1.4s
; CHECK-FP16-NEXT:    scvtf v17.4s, v3.4s
; CHECK-FP16-NEXT:    scvtf v5.4s, v5.4s
; CHECK-FP16-NEXT:    scvtf v7.4s, v7.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v2.4s
; CHECK-FP16-NEXT:    bfcvtn v2.4h, v4.4s
; CHECK-FP16-NEXT:    bfcvtn v3.4h, v6.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v16.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v17.4s
; CHECK-FP16-NEXT:    bfcvtn2 v2.8h, v5.4s
; CHECK-FP16-NEXT:    bfcvtn2 v3.8h, v7.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <32 x i32> %a to <32 x bfloat>
  ret <32 x bfloat> %c
}

define <32 x bfloat> @utofp_v32i32_v32bf16(<32 x i32> %a) {
; CHECK-NOFP16-LABEL: utofp_v32i32_v32bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    ucvtf v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    ucvtf v4.4s, v4.4s
; CHECK-NOFP16-NEXT:    ucvtf v6.4s, v6.4s
; CHECK-NOFP16-NEXT:    movi v16.4s, #1
; CHECK-NOFP16-NEXT:    ucvtf v1.4s, v1.4s
; CHECK-NOFP16-NEXT:    ucvtf v17.4s, v3.4s
; CHECK-NOFP16-NEXT:    ucvtf v5.4s, v5.4s
; CHECK-NOFP16-NEXT:    ucvtf v7.4s, v7.4s
; CHECK-NOFP16-NEXT:    movi v21.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ushr v3.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    ushr v18.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    ushr v19.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    ushr v20.4s, v6.4s, #16
; CHECK-NOFP16-NEXT:    ushr v22.4s, v1.4s, #16
; CHECK-NOFP16-NEXT:    ushr v23.4s, v17.4s, #16
; CHECK-NOFP16-NEXT:    ushr v24.4s, v5.4s, #16
; CHECK-NOFP16-NEXT:    ushr v25.4s, v7.4s, #16
; CHECK-NOFP16-NEXT:    and v3.16b, v3.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v18.16b, v18.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v19.16b, v19.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v20.16b, v20.16b, v16.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v3.4s, v0.4s
; CHECK-NOFP16-NEXT:    and v3.16b, v22.16b, v16.16b
; CHECK-NOFP16-NEXT:    add v2.4s, v18.4s, v2.4s
; CHECK-NOFP16-NEXT:    add v4.4s, v19.4s, v4.4s
; CHECK-NOFP16-NEXT:    add v6.4s, v20.4s, v6.4s
; CHECK-NOFP16-NEXT:    and v18.16b, v23.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v19.16b, v24.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v16.16b, v25.16b, v16.16b
; CHECK-NOFP16-NEXT:    add v20.4s, v3.4s, v1.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn v1.4h, v2.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn v2.4h, v4.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn v3.4h, v6.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v4.4s, v18.4s, v17.4s
; CHECK-NOFP16-NEXT:    add v5.4s, v19.4s, v5.4s
; CHECK-NOFP16-NEXT:    add v6.4s, v16.4s, v7.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v20.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn2 v1.8h, v4.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn2 v2.8h, v5.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn2 v3.8h, v6.4s, v21.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v32i32_v32bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    ucvtf v2.4s, v2.4s
; CHECK-FP16-NEXT:    ucvtf v4.4s, v4.4s
; CHECK-FP16-NEXT:    ucvtf v6.4s, v6.4s
; CHECK-FP16-NEXT:    ucvtf v16.4s, v1.4s
; CHECK-FP16-NEXT:    ucvtf v17.4s, v3.4s
; CHECK-FP16-NEXT:    ucvtf v5.4s, v5.4s
; CHECK-FP16-NEXT:    ucvtf v7.4s, v7.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v2.4s
; CHECK-FP16-NEXT:    bfcvtn v2.4h, v4.4s
; CHECK-FP16-NEXT:    bfcvtn v3.4h, v6.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v16.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v17.4s
; CHECK-FP16-NEXT:    bfcvtn2 v2.8h, v5.4s
; CHECK-FP16-NEXT:    bfcvtn2 v3.8h, v7.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <32 x i32> %a to <32 x bfloat>
  ret <32 x bfloat> %c
}

define <2 x bfloat> @stofp_v2i16_v2bf16(<2 x i16> %a) {
; CHECK-NOFP16-LABEL: stofp_v2i16_v2bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v2i16_v2bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
; CHECK-FP16-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-FP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <2 x i16> %a to <2 x bfloat>
  ret <2 x bfloat> %c
}

define <2 x bfloat> @utofp_v2i16_v2bf16(<2 x i16> %a) {
; CHECK-NOFP16-LABEL: utofp_v2i16_v2bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v2i16_v2bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
; CHECK-FP16-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-FP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <2 x i16> %a to <2 x bfloat>
  ret <2 x bfloat> %c
}

define <3 x bfloat> @stofp_v3i16_v3bf16(<3 x i16> %a) {
; CHECK-NOFP16-LABEL: stofp_v3i16_v3bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v3i16_v3bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-FP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <3 x i16> %a to <3 x bfloat>
  ret <3 x bfloat> %c
}

define <3 x bfloat> @utofp_v3i16_v3bf16(<3 x i16> %a) {
; CHECK-NOFP16-LABEL: utofp_v3i16_v3bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v3i16_v3bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-FP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <3 x i16> %a to <3 x bfloat>
  ret <3 x bfloat> %c
}

define <4 x bfloat> @stofp_v4i16_v4bf16(<4 x i16> %a) {
; CHECK-NOFP16-LABEL: stofp_v4i16_v4bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v4i16_v4bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-FP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <4 x i16> %a to <4 x bfloat>
  ret <4 x bfloat> %c
}

define <4 x bfloat> @utofp_v4i16_v4bf16(<4 x i16> %a) {
; CHECK-NOFP16-LABEL: utofp_v4i16_v4bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v4i16_v4bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-FP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <4 x i16> %a to <4 x bfloat>
  ret <4 x bfloat> %c
}

define <8 x bfloat> @stofp_v8i16_v8bf16(<8 x i16> %a) {
; CHECK-NOFP16-LABEL: stofp_v8i16_v8bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    sshll v2.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    movi v4.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    scvtf v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    scvtf v3.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v0.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    ushr v5.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    and v0.16b, v0.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v1.16b, v5.16b, v1.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v4.4s
; CHECK-NOFP16-NEXT:    add v1.4s, v1.4s, v4.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v2.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v3.4s, v1.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v8i16_v8bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    sshll v1.4s, v0.4h, #0
; CHECK-FP16-NEXT:    sshll2 v2.4s, v0.8h, #0
; CHECK-FP16-NEXT:    scvtf v1.4s, v1.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v1.4s
; CHECK-FP16-NEXT:    scvtf v1.4s, v2.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v1.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <8 x i16> %a to <8 x bfloat>
  ret <8 x bfloat> %c
}

define <8 x bfloat> @utofp_v8i16_v8bf16(<8 x i16> %a) {
; CHECK-NOFP16-LABEL: utofp_v8i16_v8bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ushll v2.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    ushll2 v0.4s, v0.8h, #0
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    movi v4.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ucvtf v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    ucvtf v3.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v0.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    ushr v5.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    and v0.16b, v0.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v1.16b, v5.16b, v1.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v4.4s
; CHECK-NOFP16-NEXT:    add v1.4s, v1.4s, v4.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v2.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v3.4s, v1.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v8i16_v8bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ushll v1.4s, v0.4h, #0
; CHECK-FP16-NEXT:    ushll2 v2.4s, v0.8h, #0
; CHECK-FP16-NEXT:    ucvtf v1.4s, v1.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v1.4s
; CHECK-FP16-NEXT:    ucvtf v1.4s, v2.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v1.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <8 x i16> %a to <8 x bfloat>
  ret <8 x bfloat> %c
}

define <16 x bfloat> @stofp_v16i16_v16bf16(<16 x i16> %a) {
; CHECK-NOFP16-LABEL: stofp_v16i16_v16bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    sshll v3.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    sshll v4.4s, v1.4h, #0
; CHECK-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
; CHECK-NOFP16-NEXT:    sshll2 v1.4s, v1.8h, #0
; CHECK-NOFP16-NEXT:    movi v2.4s, #1
; CHECK-NOFP16-NEXT:    movi v7.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    scvtf v3.4s, v3.4s
; CHECK-NOFP16-NEXT:    scvtf v4.4s, v4.4s
; CHECK-NOFP16-NEXT:    scvtf v5.4s, v0.4s
; CHECK-NOFP16-NEXT:    scvtf v6.4s, v1.4s
; CHECK-NOFP16-NEXT:    ushr v0.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    ushr v1.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    ushr v16.4s, v5.4s, #16
; CHECK-NOFP16-NEXT:    ushr v17.4s, v6.4s, #16
; CHECK-NOFP16-NEXT:    and v0.16b, v0.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v16.16b, v16.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v2.16b, v17.16b, v2.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v7.4s
; CHECK-NOFP16-NEXT:    add v1.4s, v1.4s, v7.4s
; CHECK-NOFP16-NEXT:    add v2.4s, v2.4s, v7.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v3.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v1.4h, v4.4s, v1.4s
; CHECK-NOFP16-NEXT:    add v3.4s, v16.4s, v7.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v5.4s, v3.4s
; CHECK-NOFP16-NEXT:    addhn2 v1.8h, v6.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v16i16_v16bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    sshll v2.4s, v0.4h, #0
; CHECK-FP16-NEXT:    sshll v3.4s, v1.4h, #0
; CHECK-FP16-NEXT:    sshll2 v4.4s, v0.8h, #0
; CHECK-FP16-NEXT:    sshll2 v5.4s, v1.8h, #0
; CHECK-FP16-NEXT:    scvtf v2.4s, v2.4s
; CHECK-FP16-NEXT:    scvtf v3.4s, v3.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v2.4s
; CHECK-FP16-NEXT:    scvtf v2.4s, v4.4s
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v3.4s
; CHECK-FP16-NEXT:    scvtf v3.4s, v5.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v2.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v3.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <16 x i16> %a to <16 x bfloat>
  ret <16 x bfloat> %c
}

define <16 x bfloat> @utofp_v16i16_v16bf16(<16 x i16> %a) {
; CHECK-NOFP16-LABEL: utofp_v16i16_v16bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ushll v3.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    ushll v4.4s, v1.4h, #0
; CHECK-NOFP16-NEXT:    ushll2 v0.4s, v0.8h, #0
; CHECK-NOFP16-NEXT:    ushll2 v1.4s, v1.8h, #0
; CHECK-NOFP16-NEXT:    movi v2.4s, #1
; CHECK-NOFP16-NEXT:    movi v7.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ucvtf v3.4s, v3.4s
; CHECK-NOFP16-NEXT:    ucvtf v4.4s, v4.4s
; CHECK-NOFP16-NEXT:    ucvtf v5.4s, v0.4s
; CHECK-NOFP16-NEXT:    ucvtf v6.4s, v1.4s
; CHECK-NOFP16-NEXT:    ushr v0.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    ushr v1.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    ushr v16.4s, v5.4s, #16
; CHECK-NOFP16-NEXT:    ushr v17.4s, v6.4s, #16
; CHECK-NOFP16-NEXT:    and v0.16b, v0.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v16.16b, v16.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v2.16b, v17.16b, v2.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v7.4s
; CHECK-NOFP16-NEXT:    add v1.4s, v1.4s, v7.4s
; CHECK-NOFP16-NEXT:    add v2.4s, v2.4s, v7.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v3.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v1.4h, v4.4s, v1.4s
; CHECK-NOFP16-NEXT:    add v3.4s, v16.4s, v7.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v5.4s, v3.4s
; CHECK-NOFP16-NEXT:    addhn2 v1.8h, v6.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v16i16_v16bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ushll v2.4s, v0.4h, #0
; CHECK-FP16-NEXT:    ushll v3.4s, v1.4h, #0
; CHECK-FP16-NEXT:    ushll2 v4.4s, v0.8h, #0
; CHECK-FP16-NEXT:    ushll2 v5.4s, v1.8h, #0
; CHECK-FP16-NEXT:    ucvtf v2.4s, v2.4s
; CHECK-FP16-NEXT:    ucvtf v3.4s, v3.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v2.4s
; CHECK-FP16-NEXT:    ucvtf v2.4s, v4.4s
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v3.4s
; CHECK-FP16-NEXT:    ucvtf v3.4s, v5.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v2.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v3.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <16 x i16> %a to <16 x bfloat>
  ret <16 x bfloat> %c
}

define <32 x bfloat> @stofp_v32i16_v32bf16(<32 x i16> %a) {
; CHECK-NOFP16-LABEL: stofp_v32i16_v32bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    sshll v4.4s, v1.4h, #0
; CHECK-NOFP16-NEXT:    sshll v5.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    sshll v6.4s, v2.4h, #0
; CHECK-NOFP16-NEXT:    sshll v7.4s, v3.4h, #0
; CHECK-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
; CHECK-NOFP16-NEXT:    sshll2 v1.4s, v1.8h, #0
; CHECK-NOFP16-NEXT:    sshll2 v2.4s, v2.8h, #0
; CHECK-NOFP16-NEXT:    sshll2 v3.4s, v3.8h, #0
; CHECK-NOFP16-NEXT:    movi v16.4s, #1
; CHECK-NOFP16-NEXT:    scvtf v5.4s, v5.4s
; CHECK-NOFP16-NEXT:    scvtf v4.4s, v4.4s
; CHECK-NOFP16-NEXT:    scvtf v6.4s, v6.4s
; CHECK-NOFP16-NEXT:    scvtf v7.4s, v7.4s
; CHECK-NOFP16-NEXT:    scvtf v17.4s, v0.4s
; CHECK-NOFP16-NEXT:    scvtf v18.4s, v1.4s
; CHECK-NOFP16-NEXT:    scvtf v19.4s, v2.4s
; CHECK-NOFP16-NEXT:    scvtf v20.4s, v3.4s
; CHECK-NOFP16-NEXT:    movi v21.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ushr v0.4s, v5.4s, #16
; CHECK-NOFP16-NEXT:    ushr v1.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    ushr v2.4s, v6.4s, #16
; CHECK-NOFP16-NEXT:    ushr v3.4s, v7.4s, #16
; CHECK-NOFP16-NEXT:    ushr v22.4s, v17.4s, #16
; CHECK-NOFP16-NEXT:    ushr v23.4s, v18.4s, #16
; CHECK-NOFP16-NEXT:    ushr v24.4s, v19.4s, #16
; CHECK-NOFP16-NEXT:    ushr v25.4s, v20.4s, #16
; CHECK-NOFP16-NEXT:    and v0.16b, v0.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v1.16b, v1.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v2.16b, v2.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v3.16b, v3.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v22.16b, v22.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v23.16b, v23.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v24.16b, v24.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v16.16b, v25.16b, v16.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v1.4s, v1.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v2.4s, v2.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v3.4s, v3.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v5.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v1.4h, v4.4s, v1.4s
; CHECK-NOFP16-NEXT:    addhn v2.4h, v6.4s, v2.4s
; CHECK-NOFP16-NEXT:    addhn v3.4h, v7.4s, v3.4s
; CHECK-NOFP16-NEXT:    add v4.4s, v22.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v5.4s, v23.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v6.4s, v24.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v7.4s, v16.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v17.4s, v4.4s
; CHECK-NOFP16-NEXT:    addhn2 v1.8h, v18.4s, v5.4s
; CHECK-NOFP16-NEXT:    addhn2 v2.8h, v19.4s, v6.4s
; CHECK-NOFP16-NEXT:    addhn2 v3.8h, v20.4s, v7.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v32i16_v32bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    sshll v4.4s, v0.4h, #0
; CHECK-FP16-NEXT:    sshll v5.4s, v1.4h, #0
; CHECK-FP16-NEXT:    sshll v6.4s, v2.4h, #0
; CHECK-FP16-NEXT:    sshll v7.4s, v3.4h, #0
; CHECK-FP16-NEXT:    sshll2 v0.4s, v0.8h, #0
; CHECK-FP16-NEXT:    sshll2 v1.4s, v1.8h, #0
; CHECK-FP16-NEXT:    sshll2 v2.4s, v2.8h, #0
; CHECK-FP16-NEXT:    sshll2 v3.4s, v3.8h, #0
; CHECK-FP16-NEXT:    scvtf v4.4s, v4.4s
; CHECK-FP16-NEXT:    scvtf v5.4s, v5.4s
; CHECK-FP16-NEXT:    scvtf v6.4s, v6.4s
; CHECK-FP16-NEXT:    scvtf v7.4s, v7.4s
; CHECK-FP16-NEXT:    scvtf v16.4s, v0.4s
; CHECK-FP16-NEXT:    scvtf v17.4s, v1.4s
; CHECK-FP16-NEXT:    scvtf v18.4s, v2.4s
; CHECK-FP16-NEXT:    scvtf v19.4s, v3.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v4.4s
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v5.4s
; CHECK-FP16-NEXT:    bfcvtn v2.4h, v6.4s
; CHECK-FP16-NEXT:    bfcvtn v3.4h, v7.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v16.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v17.4s
; CHECK-FP16-NEXT:    bfcvtn2 v2.8h, v18.4s
; CHECK-FP16-NEXT:    bfcvtn2 v3.8h, v19.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <32 x i16> %a to <32 x bfloat>
  ret <32 x bfloat> %c
}

define <32 x bfloat> @utofp_v32i16_v32bf16(<32 x i16> %a) {
; CHECK-NOFP16-LABEL: utofp_v32i16_v32bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ushll v4.4s, v1.4h, #0
; CHECK-NOFP16-NEXT:    ushll v5.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    ushll v6.4s, v2.4h, #0
; CHECK-NOFP16-NEXT:    ushll v7.4s, v3.4h, #0
; CHECK-NOFP16-NEXT:    ushll2 v0.4s, v0.8h, #0
; CHECK-NOFP16-NEXT:    ushll2 v1.4s, v1.8h, #0
; CHECK-NOFP16-NEXT:    ushll2 v2.4s, v2.8h, #0
; CHECK-NOFP16-NEXT:    ushll2 v3.4s, v3.8h, #0
; CHECK-NOFP16-NEXT:    movi v16.4s, #1
; CHECK-NOFP16-NEXT:    ucvtf v5.4s, v5.4s
; CHECK-NOFP16-NEXT:    ucvtf v4.4s, v4.4s
; CHECK-NOFP16-NEXT:    ucvtf v6.4s, v6.4s
; CHECK-NOFP16-NEXT:    ucvtf v7.4s, v7.4s
; CHECK-NOFP16-NEXT:    ucvtf v17.4s, v0.4s
; CHECK-NOFP16-NEXT:    ucvtf v18.4s, v1.4s
; CHECK-NOFP16-NEXT:    ucvtf v19.4s, v2.4s
; CHECK-NOFP16-NEXT:    ucvtf v20.4s, v3.4s
; CHECK-NOFP16-NEXT:    movi v21.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ushr v0.4s, v5.4s, #16
; CHECK-NOFP16-NEXT:    ushr v1.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    ushr v2.4s, v6.4s, #16
; CHECK-NOFP16-NEXT:    ushr v3.4s, v7.4s, #16
; CHECK-NOFP16-NEXT:    ushr v22.4s, v17.4s, #16
; CHECK-NOFP16-NEXT:    ushr v23.4s, v18.4s, #16
; CHECK-NOFP16-NEXT:    ushr v24.4s, v19.4s, #16
; CHECK-NOFP16-NEXT:    ushr v25.4s, v20.4s, #16
; CHECK-NOFP16-NEXT:    and v0.16b, v0.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v1.16b, v1.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v2.16b, v2.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v3.16b, v3.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v22.16b, v22.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v23.16b, v23.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v24.16b, v24.16b, v16.16b
; CHECK-NOFP16-NEXT:    and v16.16b, v25.16b, v16.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v1.4s, v1.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v2.4s, v2.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v3.4s, v3.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v5.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v1.4h, v4.4s, v1.4s
; CHECK-NOFP16-NEXT:    addhn v2.4h, v6.4s, v2.4s
; CHECK-NOFP16-NEXT:    addhn v3.4h, v7.4s, v3.4s
; CHECK-NOFP16-NEXT:    add v4.4s, v22.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v5.4s, v23.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v6.4s, v24.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v7.4s, v16.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v17.4s, v4.4s
; CHECK-NOFP16-NEXT:    addhn2 v1.8h, v18.4s, v5.4s
; CHECK-NOFP16-NEXT:    addhn2 v2.8h, v19.4s, v6.4s
; CHECK-NOFP16-NEXT:    addhn2 v3.8h, v20.4s, v7.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v32i16_v32bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ushll v4.4s, v0.4h, #0
; CHECK-FP16-NEXT:    ushll v5.4s, v1.4h, #0
; CHECK-FP16-NEXT:    ushll v6.4s, v2.4h, #0
; CHECK-FP16-NEXT:    ushll v7.4s, v3.4h, #0
; CHECK-FP16-NEXT:    ushll2 v0.4s, v0.8h, #0
; CHECK-FP16-NEXT:    ushll2 v1.4s, v1.8h, #0
; CHECK-FP16-NEXT:    ushll2 v2.4s, v2.8h, #0
; CHECK-FP16-NEXT:    ushll2 v3.4s, v3.8h, #0
; CHECK-FP16-NEXT:    ucvtf v4.4s, v4.4s
; CHECK-FP16-NEXT:    ucvtf v5.4s, v5.4s
; CHECK-FP16-NEXT:    ucvtf v6.4s, v6.4s
; CHECK-FP16-NEXT:    ucvtf v7.4s, v7.4s
; CHECK-FP16-NEXT:    ucvtf v16.4s, v0.4s
; CHECK-FP16-NEXT:    ucvtf v17.4s, v1.4s
; CHECK-FP16-NEXT:    ucvtf v18.4s, v2.4s
; CHECK-FP16-NEXT:    ucvtf v19.4s, v3.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v4.4s
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v5.4s
; CHECK-FP16-NEXT:    bfcvtn v2.4h, v6.4s
; CHECK-FP16-NEXT:    bfcvtn v3.4h, v7.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v16.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v17.4s
; CHECK-FP16-NEXT:    bfcvtn2 v2.8h, v18.4s
; CHECK-FP16-NEXT:    bfcvtn2 v3.8h, v19.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <32 x i16> %a to <32 x bfloat>
  ret <32 x bfloat> %c
}

define <2 x bfloat> @stofp_v2i8_v2bf16(<2 x i8> %a) {
; CHECK-NOFP16-LABEL: stofp_v2i8_v2bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-NOFP16-NEXT:    mov w9, v0.s[1]
; CHECK-NOFP16-NEXT:    fmov w10, s0
; CHECK-NOFP16-NEXT:    mov w8, #32767 // =0x7fff
; CHECK-NOFP16-NEXT:    sxtb w10, w10
; CHECK-NOFP16-NEXT:    sxtb w9, w9
; CHECK-NOFP16-NEXT:    scvtf s1, w10
; CHECK-NOFP16-NEXT:    scvtf s0, w9
; CHECK-NOFP16-NEXT:    fmov w10, s1
; CHECK-NOFP16-NEXT:    fmov w9, s0
; CHECK-NOFP16-NEXT:    ubfx w12, w10, #16, #1
; CHECK-NOFP16-NEXT:    ubfx w11, w9, #16, #1
; CHECK-NOFP16-NEXT:    add w9, w9, w8
; CHECK-NOFP16-NEXT:    add w8, w10, w8
; CHECK-NOFP16-NEXT:    add w8, w12, w8
; CHECK-NOFP16-NEXT:    add w9, w11, w9
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    lsr w9, w9, #16
; CHECK-NOFP16-NEXT:    fmov s0, w8
; CHECK-NOFP16-NEXT:    fmov s1, w9
; CHECK-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v2i8_v2bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-FP16-NEXT:    mov w8, v0.s[1]
; CHECK-FP16-NEXT:    fmov w9, s0
; CHECK-FP16-NEXT:    sxtb w9, w9
; CHECK-FP16-NEXT:    sxtb w8, w8
; CHECK-FP16-NEXT:    scvtf s1, w9
; CHECK-FP16-NEXT:    scvtf s0, w8
; CHECK-FP16-NEXT:    bfcvt h2, s0
; CHECK-FP16-NEXT:    bfcvt h0, s1
; CHECK-FP16-NEXT:    mov v0.h[1], v2.h[0]
; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <2 x i8> %a to <2 x bfloat>
  ret <2 x bfloat> %c
}

define <2 x bfloat> @utofp_v2i8_v2bf16(<2 x i8> %a) {
; CHECK-NOFP16-LABEL: utofp_v2i8_v2bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-NOFP16-NEXT:    mov w9, v0.s[1]
; CHECK-NOFP16-NEXT:    fmov w10, s0
; CHECK-NOFP16-NEXT:    mov w8, #32767 // =0x7fff
; CHECK-NOFP16-NEXT:    and w10, w10, #0xff
; CHECK-NOFP16-NEXT:    and w9, w9, #0xff
; CHECK-NOFP16-NEXT:    ucvtf s1, w10
; CHECK-NOFP16-NEXT:    ucvtf s0, w9
; CHECK-NOFP16-NEXT:    fmov w10, s1
; CHECK-NOFP16-NEXT:    fmov w9, s0
; CHECK-NOFP16-NEXT:    ubfx w12, w10, #16, #1
; CHECK-NOFP16-NEXT:    ubfx w11, w9, #16, #1
; CHECK-NOFP16-NEXT:    add w9, w9, w8
; CHECK-NOFP16-NEXT:    add w8, w10, w8
; CHECK-NOFP16-NEXT:    add w8, w12, w8
; CHECK-NOFP16-NEXT:    add w9, w11, w9
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    lsr w9, w9, #16
; CHECK-NOFP16-NEXT:    fmov s0, w8
; CHECK-NOFP16-NEXT:    fmov s1, w9
; CHECK-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v2i8_v2bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-FP16-NEXT:    mov w8, v0.s[1]
; CHECK-FP16-NEXT:    fmov w9, s0
; CHECK-FP16-NEXT:    and w9, w9, #0xff
; CHECK-FP16-NEXT:    and w8, w8, #0xff
; CHECK-FP16-NEXT:    ucvtf s1, w9
; CHECK-FP16-NEXT:    ucvtf s0, w8
; CHECK-FP16-NEXT:    bfcvt h2, s0
; CHECK-FP16-NEXT:    bfcvt h0, s1
; CHECK-FP16-NEXT:    mov v0.h[1], v2.h[0]
; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <2 x i8> %a to <2 x bfloat>
  ret <2 x bfloat> %c
}

define <3 x bfloat> @stofp_v3i8_v3bf16(<3 x i8> %a) {
; CHECK-NOFP16-LABEL: stofp_v3i8_v3bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    fmov s0, w0
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    mov v0.h[1], w1
; CHECK-NOFP16-NEXT:    mov v0.h[2], w2
; CHECK-NOFP16-NEXT:    shl v0.4h, v0.4h, #8
; CHECK-NOFP16-NEXT:    sshr v0.4h, v0.4h, #8
; CHECK-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v3i8_v3bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    fmov s0, w0
; CHECK-FP16-NEXT:    mov v0.h[1], w1
; CHECK-FP16-NEXT:    mov v0.h[2], w2
; CHECK-FP16-NEXT:    shl v0.4h, v0.4h, #8
; CHECK-FP16-NEXT:    sshr v0.4h, v0.4h, #8
; CHECK-FP16-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-FP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <3 x i8> %a to <3 x bfloat>
  ret <3 x bfloat> %c
}

define <3 x bfloat> @utofp_v3i8_v3bf16(<3 x i8> %a) {
; CHECK-NOFP16-LABEL: utofp_v3i8_v3bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    fmov s0, w0
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    mov v0.h[1], w1
; CHECK-NOFP16-NEXT:    mov v0.h[2], w2
; CHECK-NOFP16-NEXT:    bic v0.4h, #255, lsl #8
; CHECK-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v3i8_v3bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    fmov s0, w0
; CHECK-FP16-NEXT:    mov v0.h[1], w1
; CHECK-FP16-NEXT:    mov v0.h[2], w2
; CHECK-FP16-NEXT:    bic v0.4h, #255, lsl #8
; CHECK-FP16-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-FP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <3 x i8> %a to <3 x bfloat>
  ret <3 x bfloat> %c
}

define <4 x bfloat> @stofp_v4i8_v4bf16(<4 x i8> %a) {
; CHECK-NOFP16-LABEL: stofp_v4i8_v4bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    shl v0.4h, v0.4h, #8
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    sshr v0.4h, v0.4h, #8
; CHECK-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v4i8_v4bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    shl v0.4h, v0.4h, #8
; CHECK-FP16-NEXT:    sshr v0.4h, v0.4h, #8
; CHECK-FP16-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-FP16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <4 x i8> %a to <4 x bfloat>
  ret <4 x bfloat> %c
}

define <4 x bfloat> @utofp_v4i8_v4bf16(<4 x i8> %a) {
; CHECK-NOFP16-LABEL: utofp_v4i8_v4bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    bic v0.4h, #255, lsl #8
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-NOFP16-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-NOFP16-NEXT:    movi v2.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v4i8_v4bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    bic v0.4h, #255, lsl #8
; CHECK-FP16-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-FP16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <4 x i8> %a to <4 x bfloat>
  ret <4 x bfloat> %c
}

define <8 x bfloat> @stofp_v8i8_v8bf16(<8 x i8> %a) {
; CHECK-NOFP16-LABEL: stofp_v8i8_v8bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    movi v4.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    sshll v2.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
; CHECK-NOFP16-NEXT:    scvtf v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    scvtf v3.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v0.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    ushr v5.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    and v0.16b, v0.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v1.16b, v5.16b, v1.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v4.4s
; CHECK-NOFP16-NEXT:    add v1.4s, v1.4s, v4.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v2.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v3.4s, v1.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v8i8_v8bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-FP16-NEXT:    sshll v1.4s, v0.4h, #0
; CHECK-FP16-NEXT:    sshll2 v2.4s, v0.8h, #0
; CHECK-FP16-NEXT:    scvtf v1.4s, v1.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v1.4s
; CHECK-FP16-NEXT:    scvtf v1.4s, v2.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v1.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <8 x i8> %a to <8 x bfloat>
  ret <8 x bfloat> %c
}

define <8 x bfloat> @utofp_v8i8_v8bf16(<8 x i8> %a) {
; CHECK-NOFP16-LABEL: utofp_v8i8_v8bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    movi v4.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ushll v2.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    ushll2 v0.4s, v0.8h, #0
; CHECK-NOFP16-NEXT:    ucvtf v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    ucvtf v3.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v0.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    ushr v5.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    and v0.16b, v0.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v1.16b, v5.16b, v1.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v4.4s
; CHECK-NOFP16-NEXT:    add v1.4s, v1.4s, v4.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v2.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v3.4s, v1.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v8i8_v8bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-FP16-NEXT:    ushll v1.4s, v0.4h, #0
; CHECK-FP16-NEXT:    ushll2 v2.4s, v0.8h, #0
; CHECK-FP16-NEXT:    ucvtf v1.4s, v1.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v1.4s
; CHECK-FP16-NEXT:    ucvtf v1.4s, v2.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v1.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <8 x i8> %a to <8 x bfloat>
  ret <8 x bfloat> %c
}

define <16 x bfloat> @stofp_v16i8_v16bf16(<16 x i8> %a) {
; CHECK-NOFP16-LABEL: stofp_v16i8_v16bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    sshll2 v2.8h, v0.16b, #0
; CHECK-NOFP16-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    movi v7.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    sshll v3.4s, v2.4h, #0
; CHECK-NOFP16-NEXT:    sshll v4.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    sshll2 v2.4s, v2.8h, #0
; CHECK-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
; CHECK-NOFP16-NEXT:    scvtf v3.4s, v3.4s
; CHECK-NOFP16-NEXT:    scvtf v4.4s, v4.4s
; CHECK-NOFP16-NEXT:    scvtf v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    scvtf v6.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v5.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    ushr v0.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    ushr v16.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    ushr v17.4s, v6.4s, #16
; CHECK-NOFP16-NEXT:    and v5.16b, v5.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v0.16b, v0.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v16.16b, v16.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v17.16b, v17.16b, v1.16b
; CHECK-NOFP16-NEXT:    add v5.4s, v5.4s, v7.4s
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v7.4s
; CHECK-NOFP16-NEXT:    addhn v1.4h, v3.4s, v5.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v4.4s, v0.4s
; CHECK-NOFP16-NEXT:    add v3.4s, v16.4s, v7.4s
; CHECK-NOFP16-NEXT:    add v4.4s, v17.4s, v7.4s
; CHECK-NOFP16-NEXT:    addhn2 v1.8h, v2.4s, v3.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v6.4s, v4.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v16i8_v16bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    sshll2 v1.8h, v0.16b, #0
; CHECK-FP16-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-FP16-NEXT:    sshll v2.4s, v1.4h, #0
; CHECK-FP16-NEXT:    sshll v3.4s, v0.4h, #0
; CHECK-FP16-NEXT:    sshll2 v4.4s, v1.8h, #0
; CHECK-FP16-NEXT:    sshll2 v5.4s, v0.8h, #0
; CHECK-FP16-NEXT:    scvtf v2.4s, v2.4s
; CHECK-FP16-NEXT:    scvtf v3.4s, v3.4s
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v2.4s
; CHECK-FP16-NEXT:    scvtf v2.4s, v4.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v3.4s
; CHECK-FP16-NEXT:    scvtf v3.4s, v5.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v2.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v3.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <16 x i8> %a to <16 x bfloat>
  ret <16 x bfloat> %c
}

define <16 x bfloat> @utofp_v16i8_v16bf16(<16 x i8> %a) {
; CHECK-NOFP16-LABEL: utofp_v16i8_v16bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ushll2 v2.8h, v0.16b, #0
; CHECK-NOFP16-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-NOFP16-NEXT:    movi v1.4s, #1
; CHECK-NOFP16-NEXT:    movi v7.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ushll v3.4s, v2.4h, #0
; CHECK-NOFP16-NEXT:    ushll v4.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    ushll2 v2.4s, v2.8h, #0
; CHECK-NOFP16-NEXT:    ushll2 v0.4s, v0.8h, #0
; CHECK-NOFP16-NEXT:    ucvtf v3.4s, v3.4s
; CHECK-NOFP16-NEXT:    ucvtf v4.4s, v4.4s
; CHECK-NOFP16-NEXT:    ucvtf v2.4s, v2.4s
; CHECK-NOFP16-NEXT:    ucvtf v6.4s, v0.4s
; CHECK-NOFP16-NEXT:    ushr v5.4s, v3.4s, #16
; CHECK-NOFP16-NEXT:    ushr v0.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    ushr v16.4s, v2.4s, #16
; CHECK-NOFP16-NEXT:    ushr v17.4s, v6.4s, #16
; CHECK-NOFP16-NEXT:    and v5.16b, v5.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v0.16b, v0.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v16.16b, v16.16b, v1.16b
; CHECK-NOFP16-NEXT:    and v17.16b, v17.16b, v1.16b
; CHECK-NOFP16-NEXT:    add v5.4s, v5.4s, v7.4s
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v7.4s
; CHECK-NOFP16-NEXT:    addhn v1.4h, v3.4s, v5.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v4.4s, v0.4s
; CHECK-NOFP16-NEXT:    add v3.4s, v16.4s, v7.4s
; CHECK-NOFP16-NEXT:    add v4.4s, v17.4s, v7.4s
; CHECK-NOFP16-NEXT:    addhn2 v1.8h, v2.4s, v3.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v6.4s, v4.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v16i8_v16bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ushll2 v1.8h, v0.16b, #0
; CHECK-FP16-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-FP16-NEXT:    ushll v2.4s, v1.4h, #0
; CHECK-FP16-NEXT:    ushll v3.4s, v0.4h, #0
; CHECK-FP16-NEXT:    ushll2 v4.4s, v1.8h, #0
; CHECK-FP16-NEXT:    ushll2 v5.4s, v0.8h, #0
; CHECK-FP16-NEXT:    ucvtf v2.4s, v2.4s
; CHECK-FP16-NEXT:    ucvtf v3.4s, v3.4s
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v2.4s
; CHECK-FP16-NEXT:    ucvtf v2.4s, v4.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v3.4s
; CHECK-FP16-NEXT:    ucvtf v3.4s, v5.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v2.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v3.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <16 x i8> %a to <16 x bfloat>
  ret <16 x bfloat> %c
}

define <32 x bfloat> @stofp_v32i8_v32bf16(<32 x i8> %a) {
; CHECK-NOFP16-LABEL: stofp_v32i8_v32bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    sshll2 v3.8h, v0.16b, #0
; CHECK-NOFP16-NEXT:    sshll v0.8h, v0.8b, #0
; CHECK-NOFP16-NEXT:    sshll2 v4.8h, v1.16b, #0
; CHECK-NOFP16-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-NOFP16-NEXT:    movi v2.4s, #1
; CHECK-NOFP16-NEXT:    movi v21.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    sshll v5.4s, v3.4h, #0
; CHECK-NOFP16-NEXT:    sshll v6.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    sshll v7.4s, v4.4h, #0
; CHECK-NOFP16-NEXT:    sshll v16.4s, v1.4h, #0
; CHECK-NOFP16-NEXT:    sshll2 v3.4s, v3.8h, #0
; CHECK-NOFP16-NEXT:    sshll2 v4.4s, v4.8h, #0
; CHECK-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
; CHECK-NOFP16-NEXT:    sshll2 v1.4s, v1.8h, #0
; CHECK-NOFP16-NEXT:    scvtf v5.4s, v5.4s
; CHECK-NOFP16-NEXT:    scvtf v6.4s, v6.4s
; CHECK-NOFP16-NEXT:    scvtf v7.4s, v7.4s
; CHECK-NOFP16-NEXT:    scvtf v16.4s, v16.4s
; CHECK-NOFP16-NEXT:    scvtf v17.4s, v3.4s
; CHECK-NOFP16-NEXT:    scvtf v4.4s, v4.4s
; CHECK-NOFP16-NEXT:    scvtf v18.4s, v0.4s
; CHECK-NOFP16-NEXT:    scvtf v19.4s, v1.4s
; CHECK-NOFP16-NEXT:    ushr v0.4s, v5.4s, #16
; CHECK-NOFP16-NEXT:    ushr v3.4s, v6.4s, #16
; CHECK-NOFP16-NEXT:    ushr v1.4s, v7.4s, #16
; CHECK-NOFP16-NEXT:    ushr v20.4s, v16.4s, #16
; CHECK-NOFP16-NEXT:    ushr v23.4s, v17.4s, #16
; CHECK-NOFP16-NEXT:    ushr v24.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    ushr v22.4s, v18.4s, #16
; CHECK-NOFP16-NEXT:    ushr v25.4s, v19.4s, #16
; CHECK-NOFP16-NEXT:    and v0.16b, v0.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v3.16b, v3.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v20.16b, v20.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v23.16b, v23.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v24.16b, v24.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v22.16b, v22.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v25.16b, v25.16b, v2.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v3.4s, v3.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v26.4s, v1.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v20.4s, v20.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn v1.4h, v5.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v6.4s, v3.4s
; CHECK-NOFP16-NEXT:    addhn v3.4h, v7.4s, v26.4s
; CHECK-NOFP16-NEXT:    addhn v2.4h, v16.4s, v20.4s
; CHECK-NOFP16-NEXT:    add v5.4s, v22.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v6.4s, v23.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v7.4s, v24.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v16.4s, v25.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v18.4s, v5.4s
; CHECK-NOFP16-NEXT:    addhn2 v1.8h, v17.4s, v6.4s
; CHECK-NOFP16-NEXT:    addhn2 v3.8h, v4.4s, v7.4s
; CHECK-NOFP16-NEXT:    addhn2 v2.8h, v19.4s, v16.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: stofp_v32i8_v32bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    sshll v2.8h, v0.8b, #0
; CHECK-FP16-NEXT:    sshll2 v0.8h, v0.16b, #0
; CHECK-FP16-NEXT:    sshll2 v3.8h, v1.16b, #0
; CHECK-FP16-NEXT:    sshll v1.8h, v1.8b, #0
; CHECK-FP16-NEXT:    sshll v4.4s, v2.4h, #0
; CHECK-FP16-NEXT:    sshll v5.4s, v0.4h, #0
; CHECK-FP16-NEXT:    sshll v6.4s, v3.4h, #0
; CHECK-FP16-NEXT:    sshll v7.4s, v1.4h, #0
; CHECK-FP16-NEXT:    sshll2 v0.4s, v0.8h, #0
; CHECK-FP16-NEXT:    sshll2 v3.4s, v3.8h, #0
; CHECK-FP16-NEXT:    sshll2 v2.4s, v2.8h, #0
; CHECK-FP16-NEXT:    sshll2 v1.4s, v1.8h, #0
; CHECK-FP16-NEXT:    scvtf v4.4s, v4.4s
; CHECK-FP16-NEXT:    scvtf v5.4s, v5.4s
; CHECK-FP16-NEXT:    scvtf v6.4s, v6.4s
; CHECK-FP16-NEXT:    scvtf v7.4s, v7.4s
; CHECK-FP16-NEXT:    scvtf v16.4s, v0.4s
; CHECK-FP16-NEXT:    scvtf v17.4s, v3.4s
; CHECK-FP16-NEXT:    scvtf v18.4s, v2.4s
; CHECK-FP16-NEXT:    scvtf v19.4s, v1.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v4.4s
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v5.4s
; CHECK-FP16-NEXT:    bfcvtn v3.4h, v6.4s
; CHECK-FP16-NEXT:    bfcvtn v2.4h, v7.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v18.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v16.4s
; CHECK-FP16-NEXT:    bfcvtn2 v3.8h, v17.4s
; CHECK-FP16-NEXT:    bfcvtn2 v2.8h, v19.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = sitofp <32 x i8> %a to <32 x bfloat>
  ret <32 x bfloat> %c
}

define <32 x bfloat> @utofp_v32i8_v32bf16(<32 x i8> %a) {
; CHECK-NOFP16-LABEL: utofp_v32i8_v32bf16:
; CHECK-NOFP16:       // %bb.0: // %entry
; CHECK-NOFP16-NEXT:    ushll2 v3.8h, v0.16b, #0
; CHECK-NOFP16-NEXT:    ushll v0.8h, v0.8b, #0
; CHECK-NOFP16-NEXT:    ushll2 v4.8h, v1.16b, #0
; CHECK-NOFP16-NEXT:    ushll v1.8h, v1.8b, #0
; CHECK-NOFP16-NEXT:    movi v2.4s, #1
; CHECK-NOFP16-NEXT:    movi v21.4s, #127, msl #8
; CHECK-NOFP16-NEXT:    ushll v5.4s, v3.4h, #0
; CHECK-NOFP16-NEXT:    ushll v6.4s, v0.4h, #0
; CHECK-NOFP16-NEXT:    ushll v7.4s, v4.4h, #0
; CHECK-NOFP16-NEXT:    ushll v16.4s, v1.4h, #0
; CHECK-NOFP16-NEXT:    ushll2 v3.4s, v3.8h, #0
; CHECK-NOFP16-NEXT:    ushll2 v4.4s, v4.8h, #0
; CHECK-NOFP16-NEXT:    ushll2 v0.4s, v0.8h, #0
; CHECK-NOFP16-NEXT:    ushll2 v1.4s, v1.8h, #0
; CHECK-NOFP16-NEXT:    ucvtf v5.4s, v5.4s
; CHECK-NOFP16-NEXT:    ucvtf v6.4s, v6.4s
; CHECK-NOFP16-NEXT:    ucvtf v7.4s, v7.4s
; CHECK-NOFP16-NEXT:    ucvtf v16.4s, v16.4s
; CHECK-NOFP16-NEXT:    ucvtf v17.4s, v3.4s
; CHECK-NOFP16-NEXT:    ucvtf v4.4s, v4.4s
; CHECK-NOFP16-NEXT:    ucvtf v18.4s, v0.4s
; CHECK-NOFP16-NEXT:    ucvtf v19.4s, v1.4s
; CHECK-NOFP16-NEXT:    ushr v0.4s, v5.4s, #16
; CHECK-NOFP16-NEXT:    ushr v3.4s, v6.4s, #16
; CHECK-NOFP16-NEXT:    ushr v1.4s, v7.4s, #16
; CHECK-NOFP16-NEXT:    ushr v20.4s, v16.4s, #16
; CHECK-NOFP16-NEXT:    ushr v23.4s, v17.4s, #16
; CHECK-NOFP16-NEXT:    ushr v24.4s, v4.4s, #16
; CHECK-NOFP16-NEXT:    ushr v22.4s, v18.4s, #16
; CHECK-NOFP16-NEXT:    ushr v25.4s, v19.4s, #16
; CHECK-NOFP16-NEXT:    and v0.16b, v0.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v3.16b, v3.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v20.16b, v20.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v23.16b, v23.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v24.16b, v24.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v22.16b, v22.16b, v2.16b
; CHECK-NOFP16-NEXT:    and v25.16b, v25.16b, v2.16b
; CHECK-NOFP16-NEXT:    add v0.4s, v0.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v3.4s, v3.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v26.4s, v1.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v20.4s, v20.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn v1.4h, v5.4s, v0.4s
; CHECK-NOFP16-NEXT:    addhn v0.4h, v6.4s, v3.4s
; CHECK-NOFP16-NEXT:    addhn v3.4h, v7.4s, v26.4s
; CHECK-NOFP16-NEXT:    addhn v2.4h, v16.4s, v20.4s
; CHECK-NOFP16-NEXT:    add v5.4s, v22.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v6.4s, v23.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v7.4s, v24.4s, v21.4s
; CHECK-NOFP16-NEXT:    add v16.4s, v25.4s, v21.4s
; CHECK-NOFP16-NEXT:    addhn2 v0.8h, v18.4s, v5.4s
; CHECK-NOFP16-NEXT:    addhn2 v1.8h, v17.4s, v6.4s
; CHECK-NOFP16-NEXT:    addhn2 v3.8h, v4.4s, v7.4s
; CHECK-NOFP16-NEXT:    addhn2 v2.8h, v19.4s, v16.4s
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v32i8_v32bf16:
; CHECK-FP16:       // %bb.0: // %entry
; CHECK-FP16-NEXT:    ushll v2.8h, v0.8b, #0
; CHECK-FP16-NEXT:    ushll2 v0.8h, v0.16b, #0
; CHECK-FP16-NEXT:    ushll2 v3.8h, v1.16b, #0
; CHECK-FP16-NEXT:    ushll v1.8h, v1.8b, #0
; CHECK-FP16-NEXT:    ushll v4.4s, v2.4h, #0
; CHECK-FP16-NEXT:    ushll v5.4s, v0.4h, #0
; CHECK-FP16-NEXT:    ushll v6.4s, v3.4h, #0
; CHECK-FP16-NEXT:    ushll v7.4s, v1.4h, #0
; CHECK-FP16-NEXT:    ushll2 v0.4s, v0.8h, #0
; CHECK-FP16-NEXT:    ushll2 v3.4s, v3.8h, #0
; CHECK-FP16-NEXT:    ushll2 v2.4s, v2.8h, #0
; CHECK-FP16-NEXT:    ushll2 v1.4s, v1.8h, #0
; CHECK-FP16-NEXT:    ucvtf v4.4s, v4.4s
; CHECK-FP16-NEXT:    ucvtf v5.4s, v5.4s
; CHECK-FP16-NEXT:    ucvtf v6.4s, v6.4s
; CHECK-FP16-NEXT:    ucvtf v7.4s, v7.4s
; CHECK-FP16-NEXT:    ucvtf v16.4s, v0.4s
; CHECK-FP16-NEXT:    ucvtf v17.4s, v3.4s
; CHECK-FP16-NEXT:    ucvtf v18.4s, v2.4s
; CHECK-FP16-NEXT:    ucvtf v19.4s, v1.4s
; CHECK-FP16-NEXT:    bfcvtn v0.4h, v4.4s
; CHECK-FP16-NEXT:    bfcvtn v1.4h, v5.4s
; CHECK-FP16-NEXT:    bfcvtn v3.4h, v6.4s
; CHECK-FP16-NEXT:    bfcvtn v2.4h, v7.4s
; CHECK-FP16-NEXT:    bfcvtn2 v0.8h, v18.4s
; CHECK-FP16-NEXT:    bfcvtn2 v1.8h, v16.4s
; CHECK-FP16-NEXT:    bfcvtn2 v3.8h, v17.4s
; CHECK-FP16-NEXT:    bfcvtn2 v2.8h, v19.4s
; CHECK-FP16-NEXT:    ret
entry:
  %c = uitofp <32 x i8> %a to <32 x bfloat>
  ret <32 x bfloat> %c
}

define <4 x bfloat> @utofp_v4i48_v4f16(<4 x i48> %a) {
; CHECK-NOFP16-LABEL: utofp_v4i48_v4f16:
; CHECK-NOFP16:       // %bb.0:
; CHECK-NOFP16-NEXT:    and x8, x1, #0xffffffffffff
; CHECK-NOFP16-NEXT:    and x9, x0, #0xffffffffffff
; CHECK-NOFP16-NEXT:    ucvtf d0, x8
; CHECK-NOFP16-NEXT:    ucvtf d1, x9
; CHECK-NOFP16-NEXT:    and x8, x2, #0xffffffffffff
; CHECK-NOFP16-NEXT:    ucvtf d2, x8
; CHECK-NOFP16-NEXT:    and x8, x3, #0xffffffffffff
; CHECK-NOFP16-NEXT:    mov w9, #32767 // =0x7fff
; CHECK-NOFP16-NEXT:    fcvtxn s0, d0
; CHECK-NOFP16-NEXT:    fcvtxn s1, d1
; CHECK-NOFP16-NEXT:    fcvtxn s2, d2
; CHECK-NOFP16-NEXT:    fmov w10, s0
; CHECK-NOFP16-NEXT:    ucvtf d0, x8
; CHECK-NOFP16-NEXT:    fmov w8, s1
; CHECK-NOFP16-NEXT:    ubfx w11, w10, #16, #1
; CHECK-NOFP16-NEXT:    add w10, w10, w9
; CHECK-NOFP16-NEXT:    ubfx w12, w8, #16, #1
; CHECK-NOFP16-NEXT:    add w8, w8, w9
; CHECK-NOFP16-NEXT:    fcvtxn s1, d0
; CHECK-NOFP16-NEXT:    add w10, w11, w10
; CHECK-NOFP16-NEXT:    fmov w11, s2
; CHECK-NOFP16-NEXT:    add w8, w12, w8
; CHECK-NOFP16-NEXT:    lsr w10, w10, #16
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    ubfx w12, w11, #16, #1
; CHECK-NOFP16-NEXT:    fmov s2, w10
; CHECK-NOFP16-NEXT:    fmov s0, w8
; CHECK-NOFP16-NEXT:    add w8, w11, w9
; CHECK-NOFP16-NEXT:    fmov w10, s1
; CHECK-NOFP16-NEXT:    add w8, w12, w8
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    mov v0.h[1], v2.h[0]
; CHECK-NOFP16-NEXT:    ubfx w11, w10, #16, #1
; CHECK-NOFP16-NEXT:    fmov s1, w8
; CHECK-NOFP16-NEXT:    add w8, w10, w9
; CHECK-NOFP16-NEXT:    add w8, w11, w8
; CHECK-NOFP16-NEXT:    lsr w8, w8, #16
; CHECK-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
; CHECK-NOFP16-NEXT:    fmov s1, w8
; CHECK-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
; CHECK-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: utofp_v4i48_v4f16:
; CHECK-FP16:       // %bb.0:
; CHECK-FP16-NEXT:    and x8, x1, #0xffffffffffff
; CHECK-FP16-NEXT:    and x9, x0, #0xffffffffffff
; CHECK-FP16-NEXT:    ucvtf d0, x8
; CHECK-FP16-NEXT:    ucvtf d1, x9
; CHECK-FP16-NEXT:    and x8, x2, #0xffffffffffff
; CHECK-FP16-NEXT:    ucvtf d2, x8
; CHECK-FP16-NEXT:    and x8, x3, #0xffffffffffff
; CHECK-FP16-NEXT:    fcvtxn s0, d0
; CHECK-FP16-NEXT:    fcvtxn s1, d1
; CHECK-FP16-NEXT:    bfcvt h3, s0
; CHECK-FP16-NEXT:    bfcvt h0, s1
; CHECK-FP16-NEXT:    fcvtxn s1, d2
; CHECK-FP16-NEXT:    ucvtf d2, x8
; CHECK-FP16-NEXT:    mov v0.h[1], v3.h[0]
; CHECK-FP16-NEXT:    bfcvt h1, s1
; CHECK-FP16-NEXT:    fcvtxn s2, d2
; CHECK-FP16-NEXT:    mov v0.h[2], v1.h[0]
; CHECK-FP16-NEXT:    bfcvt h1, s2
; CHECK-FP16-NEXT:    mov v0.h[3], v1.h[0]
; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-FP16-NEXT:    ret
  %r = uitofp <4 x i48> %a to <4 x bfloat>
  ret <4 x bfloat> %r
}

;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK: {{.*}}
; CHECK-FP16-GI: {{.*}}
; CHECK-FP16-SD: {{.*}}
; CHECK-NOFP16-GI: {{.*}}
; CHECK-NOFP16-SD: {{.*}}