test/CodeGen/AArch64/bf16-v4-instructions.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=aarch64 -mattr=-bf16 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT
; RUN: llc < %s -mtriple=aarch64 -mattr=+bf16 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16

define <4 x bfloat> @add_h(<4 x bfloat> %a, <4 x bfloat> %b) {
; CHECK-CVT-LABEL: add_h:
; CHECK-CVT:       // %bb.0: // %entry
; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-CVT-NEXT:    fadd v0.4s, v0.4s, v1.4s
; CHECK-CVT-NEXT:    movi v1.4s, #1
; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-CVT-NEXT:    ret
;
; CHECK-BF16-LABEL: add_h:
; CHECK-BF16:       // %bb.0: // %entry
; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-BF16-NEXT:    fadd v0.4s, v0.4s, v1.4s
; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT:    ret
entry:

  %0 = fadd <4 x bfloat> %a, %b
  ret <4 x bfloat> %0
}


define <4 x bfloat> @build_h4(<4 x bfloat> %a) {
; CHECK-LABEL: build_h4:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov w8, #15565 // =0x3ccd
; CHECK-NEXT:    dup v0.4h, w8
; CHECK-NEXT:    ret
entry:
  ret <4 x bfloat> <bfloat 0xR3CCD, bfloat 0xR3CCD, bfloat 0xR3CCD, bfloat 0xR3CCD>
}


define <4 x bfloat> @sub_h(<4 x bfloat> %a, <4 x bfloat> %b) {
; CHECK-CVT-LABEL: sub_h:
; CHECK-CVT:       // %bb.0: // %entry
; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-CVT-NEXT:    fsub v0.4s, v0.4s, v1.4s
; CHECK-CVT-NEXT:    movi v1.4s, #1
; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-CVT-NEXT:    ret
;
; CHECK-BF16-LABEL: sub_h:
; CHECK-BF16:       // %bb.0: // %entry
; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-BF16-NEXT:    fsub v0.4s, v0.4s, v1.4s
; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT:    ret
entry:

  %0 = fsub <4 x bfloat> %a, %b
  ret <4 x bfloat> %0
}


define <4 x bfloat> @mul_h(<4 x bfloat> %a, <4 x bfloat> %b) {
; CHECK-CVT-LABEL: mul_h:
; CHECK-CVT:       // %bb.0: // %entry
; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-CVT-NEXT:    fmul v0.4s, v0.4s, v1.4s
; CHECK-CVT-NEXT:    movi v1.4s, #1
; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-CVT-NEXT:    ret
;
; CHECK-BF16-LABEL: mul_h:
; CHECK-BF16:       // %bb.0: // %entry
; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-BF16-NEXT:    fmul v0.4s, v0.4s, v1.4s
; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT:    ret
entry:

  %0 = fmul <4 x bfloat> %a, %b
  ret <4 x bfloat> %0
}


define <4 x bfloat> @div_h(<4 x bfloat> %a, <4 x bfloat> %b) {
; CHECK-CVT-LABEL: div_h:
; CHECK-CVT:       // %bb.0: // %entry
; CHECK-CVT-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-CVT-NEXT:    fdiv v0.4s, v0.4s, v1.4s
; CHECK-CVT-NEXT:    movi v1.4s, #1
; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-CVT-NEXT:    ret
;
; CHECK-BF16-LABEL: div_h:
; CHECK-BF16:       // %bb.0: // %entry
; CHECK-BF16-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-BF16-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-BF16-NEXT:    fdiv v0.4s, v0.4s, v1.4s
; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT:    ret
entry:

  %0 = fdiv <4 x bfloat> %a, %b
  ret <4 x bfloat> %0
}


define <4 x bfloat> @load_h(ptr %a) {
; CHECK-LABEL: load_h:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ret
entry:
  %0 = load <4 x bfloat>, ptr %a, align 4
  ret <4 x bfloat> %0
}


define void @store_h(ptr %a, <4 x bfloat> %b) {
; CHECK-LABEL: store_h:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str d0, [x0]
; CHECK-NEXT:    ret
entry:
  store <4 x bfloat> %b, ptr %a, align 4
  ret void
}

define <4 x bfloat> @s_to_h(<4 x float> %a) {
; CHECK-CVT-LABEL: s_to_h:
; CHECK-CVT:       // %bb.0:
; CHECK-CVT-NEXT:    movi v1.4s, #1
; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
; CHECK-CVT-NEXT:    ushr v3.4s, v0.4s, #16
; CHECK-CVT-NEXT:    and v1.16b, v3.16b, v1.16b
; CHECK-CVT-NEXT:    add v2.4s, v0.4s, v2.4s
; CHECK-CVT-NEXT:    fcmeq v3.4s, v0.4s, v0.4s
; CHECK-CVT-NEXT:    orr v0.4s, #64, lsl #16
; CHECK-CVT-NEXT:    add v1.4s, v1.4s, v2.4s
; CHECK-CVT-NEXT:    bit v0.16b, v1.16b, v3.16b
; CHECK-CVT-NEXT:    shrn v0.4h, v0.4s, #16
; CHECK-CVT-NEXT:    ret
;
; CHECK-BF16-LABEL: s_to_h:
; CHECK-BF16:       // %bb.0:
; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT:    ret
  %1 = fptrunc <4 x float> %a to <4 x bfloat>
  ret <4 x bfloat> %1
}

define <4 x bfloat> @d_to_h(<4 x double> %a) {
; CHECK-CVT-LABEL: d_to_h:
; CHECK-CVT:       // %bb.0:
; CHECK-CVT-NEXT:    fcvtxn v0.2s, v0.2d
; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
; CHECK-CVT-NEXT:    fcvtxn2 v0.4s, v1.2d
; CHECK-CVT-NEXT:    movi v1.4s, #1
; CHECK-CVT-NEXT:    ushr v3.4s, v0.4s, #16
; CHECK-CVT-NEXT:    add v2.4s, v0.4s, v2.4s
; CHECK-CVT-NEXT:    and v1.16b, v3.16b, v1.16b
; CHECK-CVT-NEXT:    fcmeq v3.4s, v0.4s, v0.4s
; CHECK-CVT-NEXT:    orr v0.4s, #64, lsl #16
; CHECK-CVT-NEXT:    add v1.4s, v1.4s, v2.4s
; CHECK-CVT-NEXT:    bit v0.16b, v1.16b, v3.16b
; CHECK-CVT-NEXT:    shrn v0.4h, v0.4s, #16
; CHECK-CVT-NEXT:    ret
;
; CHECK-BF16-LABEL: d_to_h:
; CHECK-BF16:       // %bb.0:
; CHECK-BF16-NEXT:    fcvtxn v0.2s, v0.2d
; CHECK-BF16-NEXT:    fcvtxn2 v0.4s, v1.2d
; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT:    ret
  %1 = fptrunc <4 x double> %a to <4 x bfloat>
  ret <4 x bfloat> %1
}

define <4 x float> @h_to_s(<4 x bfloat> %a) {
; CHECK-LABEL: h_to_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    ret
  %1 = fpext <4 x bfloat> %a to <4 x float>
  ret <4 x float> %1
}

define <4 x double> @h_to_d(<4 x bfloat> %a) {
; CHECK-LABEL: h_to_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvtl2 v1.2d, v0.4s
; CHECK-NEXT:    fcvtl v0.2d, v0.2s
; CHECK-NEXT:    ret
  %1 = fpext <4 x bfloat> %a to <4 x double>
  ret <4 x double> %1
}

define <4 x bfloat> @bitcast_i_to_h(float, <4 x i16> %a) {
; CHECK-LABEL: bitcast_i_to_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    fmov d0, d1
; CHECK-NEXT:    ret
  %2 = bitcast <4 x i16> %a to <4 x bfloat>
  ret <4 x bfloat> %2
}

define <4 x i16> @bitcast_h_to_i(float, <4 x bfloat> %a) {
; CHECK-LABEL: bitcast_h_to_i:
; CHECK:       // %bb.0:
; CHECK-NEXT:    fmov d0, d1
; CHECK-NEXT:    ret
  %2 = bitcast <4 x bfloat> %a to <4 x i16>
  ret <4 x i16> %2
}

define <4 x bfloat> @sitofp_i8(<4 x i8> %a) #0 {
; CHECK-CVT-LABEL: sitofp_i8:
; CHECK-CVT:       // %bb.0:
; CHECK-CVT-NEXT:    shl v0.4h, v0.4h, #8
; CHECK-CVT-NEXT:    movi v1.4s, #1
; CHECK-CVT-NEXT:    sshr v0.4h, v0.4h, #8
; CHECK-CVT-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-CVT-NEXT:    scvtf v0.4s, v0.4s
; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-CVT-NEXT:    ret
;
; CHECK-BF16-LABEL: sitofp_i8:
; CHECK-BF16:       // %bb.0:
; CHECK-BF16-NEXT:    shl v0.4h, v0.4h, #8
; CHECK-BF16-NEXT:    sshr v0.4h, v0.4h, #8
; CHECK-BF16-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-BF16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT:    ret
  %1 = sitofp <4 x i8> %a to <4 x bfloat>
  ret <4 x bfloat> %1
}

define <4 x bfloat> @sitofp_i16(<4 x i16> %a) #0 {
; CHECK-CVT-LABEL: sitofp_i16:
; CHECK-CVT:       // %bb.0:
; CHECK-CVT-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-CVT-NEXT:    movi v1.4s, #1
; CHECK-CVT-NEXT:    scvtf v0.4s, v0.4s
; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-CVT-NEXT:    ret
;
; CHECK-BF16-LABEL: sitofp_i16:
; CHECK-BF16:       // %bb.0:
; CHECK-BF16-NEXT:    sshll v0.4s, v0.4h, #0
; CHECK-BF16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT:    ret
  %1 = sitofp <4 x i16> %a to <4 x bfloat>
  ret <4 x bfloat> %1
}


define <4 x bfloat> @sitofp_i32(<4 x i32> %a) #0 {
; CHECK-CVT-LABEL: sitofp_i32:
; CHECK-CVT:       // %bb.0:
; CHECK-CVT-NEXT:    scvtf v0.4s, v0.4s
; CHECK-CVT-NEXT:    movi v1.4s, #1
; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-CVT-NEXT:    ret
;
; CHECK-BF16-LABEL: sitofp_i32:
; CHECK-BF16:       // %bb.0:
; CHECK-BF16-NEXT:    scvtf v0.4s, v0.4s
; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT:    ret
  %1 = sitofp <4 x i32> %a to <4 x bfloat>
  ret <4 x bfloat> %1
}


define <4 x bfloat> @sitofp_i64(<4 x i64> %a) #0 {
; CHECK-CVT-LABEL: sitofp_i64:
; CHECK-CVT:       // %bb.0:
; CHECK-CVT-NEXT:    mov x8, v0.d[1]
; CHECK-CVT-NEXT:    fmov x9, d0
; CHECK-CVT-NEXT:    scvtf s2, x9
; CHECK-CVT-NEXT:    mov x9, v1.d[1]
; CHECK-CVT-NEXT:    scvtf s0, x8
; CHECK-CVT-NEXT:    fmov x8, d1
; CHECK-CVT-NEXT:    scvtf s1, x8
; CHECK-CVT-NEXT:    mov v2.s[1], v0.s[0]
; CHECK-CVT-NEXT:    scvtf s0, x9
; CHECK-CVT-NEXT:    mov v2.s[2], v1.s[0]
; CHECK-CVT-NEXT:    movi v1.4s, #127, msl #8
; CHECK-CVT-NEXT:    mov v2.s[3], v0.s[0]
; CHECK-CVT-NEXT:    movi v0.4s, #1
; CHECK-CVT-NEXT:    ushr v3.4s, v2.4s, #16
; CHECK-CVT-NEXT:    add v1.4s, v2.4s, v1.4s
; CHECK-CVT-NEXT:    and v0.16b, v3.16b, v0.16b
; CHECK-CVT-NEXT:    fcmeq v3.4s, v2.4s, v2.4s
; CHECK-CVT-NEXT:    orr v2.4s, #64, lsl #16
; CHECK-CVT-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-CVT-NEXT:    bif v0.16b, v2.16b, v3.16b
; CHECK-CVT-NEXT:    shrn v0.4h, v0.4s, #16
; CHECK-CVT-NEXT:    ret
;
; CHECK-BF16-LABEL: sitofp_i64:
; CHECK-BF16:       // %bb.0:
; CHECK-BF16-NEXT:    mov x8, v0.d[1]
; CHECK-BF16-NEXT:    fmov x9, d0
; CHECK-BF16-NEXT:    scvtf s2, x9
; CHECK-BF16-NEXT:    mov x9, v1.d[1]
; CHECK-BF16-NEXT:    scvtf s0, x8
; CHECK-BF16-NEXT:    fmov x8, d1
; CHECK-BF16-NEXT:    mov v2.s[1], v0.s[0]
; CHECK-BF16-NEXT:    scvtf s0, x8
; CHECK-BF16-NEXT:    mov v2.s[2], v0.s[0]
; CHECK-BF16-NEXT:    scvtf s0, x9
; CHECK-BF16-NEXT:    mov v2.s[3], v0.s[0]
; CHECK-BF16-NEXT:    bfcvtn v0.4h, v2.4s
; CHECK-BF16-NEXT:    ret
  %1 = sitofp <4 x i64> %a to <4 x bfloat>
  ret <4 x bfloat> %1
}

define <4 x bfloat> @uitofp_i8(<4 x i8> %a) #0 {
; CHECK-CVT-LABEL: uitofp_i8:
; CHECK-CVT:       // %bb.0:
; CHECK-CVT-NEXT:    bic v0.4h, #255, lsl #8
; CHECK-CVT-NEXT:    movi v1.4s, #1
; CHECK-CVT-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-CVT-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-CVT-NEXT:    ret
;
; CHECK-BF16-LABEL: uitofp_i8:
; CHECK-BF16:       // %bb.0:
; CHECK-BF16-NEXT:    bic v0.4h, #255, lsl #8
; CHECK-BF16-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-BF16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT:    ret
  %1 = uitofp <4 x i8> %a to <4 x bfloat>
  ret <4 x bfloat> %1
}


define <4 x bfloat> @uitofp_i16(<4 x i16> %a) #0 {
; CHECK-CVT-LABEL: uitofp_i16:
; CHECK-CVT:       // %bb.0:
; CHECK-CVT-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-CVT-NEXT:    movi v1.4s, #1
; CHECK-CVT-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-CVT-NEXT:    ret
;
; CHECK-BF16-LABEL: uitofp_i16:
; CHECK-BF16:       // %bb.0:
; CHECK-BF16-NEXT:    ushll v0.4s, v0.4h, #0
; CHECK-BF16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT:    ret
  %1 = uitofp <4 x i16> %a to <4 x bfloat>
  ret <4 x bfloat> %1
}


define <4 x bfloat> @uitofp_i32(<4 x i32> %a) #0 {
; CHECK-CVT-LABEL: uitofp_i32:
; CHECK-CVT:       // %bb.0:
; CHECK-CVT-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-CVT-NEXT:    movi v1.4s, #1
; CHECK-CVT-NEXT:    ushr v2.4s, v0.4s, #16
; CHECK-CVT-NEXT:    and v1.16b, v2.16b, v1.16b
; CHECK-CVT-NEXT:    movi v2.4s, #127, msl #8
; CHECK-CVT-NEXT:    add v0.4s, v1.4s, v0.4s
; CHECK-CVT-NEXT:    addhn v0.4h, v0.4s, v2.4s
; CHECK-CVT-NEXT:    ret
;
; CHECK-BF16-LABEL: uitofp_i32:
; CHECK-BF16:       // %bb.0:
; CHECK-BF16-NEXT:    ucvtf v0.4s, v0.4s
; CHECK-BF16-NEXT:    bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT:    ret
  %1 = uitofp <4 x i32> %a to <4 x bfloat>
  ret <4 x bfloat> %1
}


define <4 x bfloat> @uitofp_i64(<4 x i64> %a) #0 {
; CHECK-CVT-LABEL: uitofp_i64:
; CHECK-CVT:       // %bb.0:
; CHECK-CVT-NEXT:    mov x8, v0.d[1]
; CHECK-CVT-NEXT:    fmov x9, d0
; CHECK-CVT-NEXT:    ucvtf s2, x9
; CHECK-CVT-NEXT:    mov x9, v1.d[1]
; CHECK-CVT-NEXT:    ucvtf s0, x8
; CHECK-CVT-NEXT:    fmov x8, d1
; CHECK-CVT-NEXT:    ucvtf s1, x8
; CHECK-CVT-NEXT:    mov v2.s[1], v0.s[0]
; CHECK-CVT-NEXT:    ucvtf s0, x9
; CHECK-CVT-NEXT:    mov v2.s[2], v1.s[0]
; CHECK-CVT-NEXT:    movi v1.4s, #127, msl #8
; CHECK-CVT-NEXT:    mov v2.s[3], v0.s[0]
; CHECK-CVT-NEXT:    movi v0.4s, #1
; CHECK-CVT-NEXT:    ushr v3.4s, v2.4s, #16
; CHECK-CVT-NEXT:    add v1.4s, v2.4s, v1.4s
; CHECK-CVT-NEXT:    and v0.16b, v3.16b, v0.16b
; CHECK-CVT-NEXT:    fcmeq v3.4s, v2.4s, v2.4s
; CHECK-CVT-NEXT:    orr v2.4s, #64, lsl #16
; CHECK-CVT-NEXT:    add v0.4s, v0.4s, v1.4s
; CHECK-CVT-NEXT:    bif v0.16b, v2.16b, v3.16b
; CHECK-CVT-NEXT:    shrn v0.4h, v0.4s, #16
; CHECK-CVT-NEXT:    ret
;
; CHECK-BF16-LABEL: uitofp_i64:
; CHECK-BF16:       // %bb.0:
; CHECK-BF16-NEXT:    mov x8, v0.d[1]
; CHECK-BF16-NEXT:    fmov x9, d0
; CHECK-BF16-NEXT:    ucvtf s2, x9
; CHECK-BF16-NEXT:    mov x9, v1.d[1]
; CHECK-BF16-NEXT:    ucvtf s0, x8
; CHECK-BF16-NEXT:    fmov x8, d1
; CHECK-BF16-NEXT:    mov v2.s[1], v0.s[0]
; CHECK-BF16-NEXT:    ucvtf s0, x8
; CHECK-BF16-NEXT:    mov v2.s[2], v0.s[0]
; CHECK-BF16-NEXT:    ucvtf s0, x9
; CHECK-BF16-NEXT:    mov v2.s[3], v0.s[0]
; CHECK-BF16-NEXT:    bfcvtn v0.4h, v2.4s
; CHECK-BF16-NEXT:    ret
  %1 = uitofp <4 x i64> %a to <4 x bfloat>
  ret <4 x bfloat> %1
}

define void @test_insert_at_zero(bfloat %a, ptr %b) #0 {
; CHECK-LABEL: test_insert_at_zero:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
; CHECK-NEXT:    str d0, [x0]
; CHECK-NEXT:    ret
  %1 = insertelement <4 x bfloat> undef, bfloat %a, i64 0
  store <4 x bfloat> %1, ptr %b, align 4
  ret void
}

define <4 x i8> @fptosi_i8(<4 x bfloat> %a) #0 {
; CHECK-LABEL: fptosi_i8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret
  %1 = fptosi<4 x bfloat> %a to <4 x i8>
  ret <4 x i8> %1
}

define <4 x i16> @fptosi_i16(<4 x bfloat> %a) #0 {
; CHECK-LABEL: fptosi_i16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret
  %1 = fptosi<4 x bfloat> %a to <4 x i16>
  ret <4 x i16> %1
}

define <4 x i8> @fptoui_i8(<4 x bfloat> %a) #0 {
; CHECK-LABEL: fptoui_i8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret
; NOTE: fcvtzs selected here because the xtn shaves the sign bit
  %1 = fptoui<4 x bfloat> %a to <4 x i8>
  ret <4 x i8> %1
}

define <4 x i16> @fptoui_i16(<4 x bfloat> %a) #0 {
; CHECK-LABEL: fptoui_i16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret
  %1 = fptoui<4 x bfloat> %a to <4 x i16>
  ret <4 x i16> %1
}

define <4 x i1> @test_fcmp_une(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-LABEL: test_fcmp_une:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
; CHECK-NEXT:    mvn v0.16b, v0.16b
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret

  %1 = fcmp une <4 x bfloat> %a, %b
  ret <4 x i1> %1
}

define <4 x i1> @test_fcmp_ueq(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-LABEL: test_fcmp_ueq:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcmgt v2.4s, v0.4s, v1.4s
; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    mvn v0.8b, v0.8b
; CHECK-NEXT:    ret

  %1 = fcmp ueq <4 x bfloat> %a, %b
  ret <4 x i1> %1
}

define <4 x i1> @test_fcmp_ugt(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-LABEL: test_fcmp_ugt:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-NEXT:    fcmge v0.4s, v1.4s, v0.4s
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    mvn v0.8b, v0.8b
; CHECK-NEXT:    ret

  %1 = fcmp ugt <4 x bfloat> %a, %b
  ret <4 x i1> %1
}

define <4 x i1> @test_fcmp_uge(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-LABEL: test_fcmp_uge:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    mvn v0.8b, v0.8b
; CHECK-NEXT:    ret

  %1 = fcmp uge <4 x bfloat> %a, %b
  ret <4 x i1> %1
}

define <4 x i1> @test_fcmp_ult(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-LABEL: test_fcmp_ult:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcmge v0.4s, v0.4s, v1.4s
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    mvn v0.8b, v0.8b
; CHECK-NEXT:    ret

  %1 = fcmp ult <4 x bfloat> %a, %b
  ret <4 x i1> %1
}

define <4 x i1> @test_fcmp_ule(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-LABEL: test_fcmp_ule:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    mvn v0.8b, v0.8b
; CHECK-NEXT:    ret

  %1 = fcmp ule <4 x bfloat> %a, %b
  ret <4 x i1> %1
}

define <4 x i1> @test_fcmp_uno(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-LABEL: test_fcmp_uno:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcmge v2.4s, v0.4s, v1.4s
; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    mvn v0.8b, v0.8b
; CHECK-NEXT:    ret

  %1 = fcmp uno <4 x bfloat> %a, %b
  ret <4 x i1> %1
}

define <4 x i1> @test_fcmp_one(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-LABEL: test_fcmp_one:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcmgt v2.4s, v0.4s, v1.4s
; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret

  %1 = fcmp one <4 x bfloat> %a, %b
  ret <4 x i1> %1
}

define <4 x i1> @test_fcmp_oeq(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-LABEL: test_fcmp_oeq:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret

  %1 = fcmp oeq <4 x bfloat> %a, %b
  ret <4 x i1> %1
}

define <4 x i1> @test_fcmp_ogt(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-LABEL: test_fcmp_ogt:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret

  %1 = fcmp ogt <4 x bfloat> %a, %b
  ret <4 x i1> %1
}

define <4 x i1> @test_fcmp_oge(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-LABEL: test_fcmp_oge:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcmge v0.4s, v0.4s, v1.4s
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret

  %1 = fcmp oge <4 x bfloat> %a, %b
  ret <4 x i1> %1
}

define <4 x i1> @test_fcmp_olt(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-LABEL: test_fcmp_olt:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret

  %1 = fcmp olt <4 x bfloat> %a, %b
  ret <4 x i1> %1
}

define <4 x i1> @test_fcmp_ole(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-LABEL: test_fcmp_ole:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-NEXT:    fcmge v0.4s, v1.4s, v0.4s
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret

  %1 = fcmp ole <4 x bfloat> %a, %b
  ret <4 x i1> %1
}

define <4 x i1> @test_fcmp_ord(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; CHECK-LABEL: test_fcmp_ord:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shll v1.4s, v1.4h, #16
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcmge v2.4s, v0.4s, v1.4s
; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    xtn v0.4h, v0.4s
; CHECK-NEXT:    ret

  %1 = fcmp ord <4 x bfloat> %a, %b
  ret <4 x i1> %1
}

attributes #0 = { nounwind }