llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-bf16.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=256  < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=512  < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2048
; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=256  < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=512  < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2048

define void @fcvt_v2bf16_v2f32(ptr %a, ptr %b) vscale_range(2,0) {
; CHECK-LABEL: fcvt_v2bf16_v2f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr s0, [x0]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str d0, [x1]
; CHECK-NEXT:    ret
  %op1 = load <2 x bfloat>, ptr %a
  %res = fpext <2 x bfloat> %op1 to <2 x float>
  store <2 x float> %res, ptr %b
  ret void
}

define void @fcvt_v4bf16_v4f32(ptr %a, ptr %b) vscale_range(2,0) {
; CHECK-LABEL: fcvt_v4bf16_v4f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str q0, [x1]
; CHECK-NEXT:    ret
  %op1 = load <4 x bfloat>, ptr %a
  %res = fpext <4 x bfloat> %op1 to <4 x float>
  store <4 x float> %res, ptr %b
  ret void
}

define void @fcvt_v8bf16_v8f32(ptr %a, ptr %b) vscale_range(2,0) {
; CHECK-LABEL: fcvt_v8bf16_v8f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; CHECK-NEXT:    sub x9, sp, #48
; CHECK-NEXT:    mov x29, sp
; CHECK-NEXT:    and sp, x9, #0xffffffffffffffe0
; CHECK-NEXT:    .cfi_def_cfa w29, 16
; CHECK-NEXT:    .cfi_offset w30, -8
; CHECK-NEXT:    .cfi_offset w29, -16
; CHECK-NEXT:    ldr h0, [x0, #14]
; CHECK-NEXT:    ptrue p0.s, vl8
; CHECK-NEXT:    mov x8, sp
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #28]
; CHECK-NEXT:    ldr h0, [x0, #12]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #24]
; CHECK-NEXT:    ldr h0, [x0, #10]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #20]
; CHECK-NEXT:    ldr h0, [x0, #8]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #16]
; CHECK-NEXT:    ldr h0, [x0, #6]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #12]
; CHECK-NEXT:    ldr h0, [x0, #4]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #8]
; CHECK-NEXT:    ldr h0, [x0, #2]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #4]
; CHECK-NEXT:    ldr h0, [x0]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp]
; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x8]
; CHECK-NEXT:    st1w { z0.s }, p0, [x1]
; CHECK-NEXT:    mov sp, x29
; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  %op1 = load <8 x bfloat>, ptr %a
  %res = fpext <8 x bfloat> %op1 to <8 x float>
  store <8 x float> %res, ptr %b
  ret void
}

define void @fcvt_v16bf16_v16f32(ptr %a, ptr %b) {
; VBITS_GE_256-LABEL: fcvt_v16bf16_v16f32:
; VBITS_GE_256:       // %bb.0:
; VBITS_GE_256-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; VBITS_GE_256-NEXT:    sub x9, sp, #80
; VBITS_GE_256-NEXT:    mov x29, sp
; VBITS_GE_256-NEXT:    and sp, x9, #0xffffffffffffffe0
; VBITS_GE_256-NEXT:    .cfi_def_cfa w29, 16
; VBITS_GE_256-NEXT:    .cfi_offset w30, -8
; VBITS_GE_256-NEXT:    .cfi_offset w29, -16
; VBITS_GE_256-NEXT:    ldr h0, [x0, #14]
; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
; VBITS_GE_256-NEXT:    add x8, sp, #32
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #28]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #12]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #24]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #10]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #20]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #8]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #16]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #6]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #12]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #4]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #8]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #2]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #4]
; VBITS_GE_256-NEXT:    ldr h0, [x0]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #30]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #60]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #28]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #56]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #26]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #52]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #24]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #48]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #22]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #44]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #20]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #40]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #18]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #36]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #16]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    str s0, [sp, #32]
; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x8]
; VBITS_GE_256-NEXT:    mov x8, sp
; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x8]
; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1]
; VBITS_GE_256-NEXT:    mov sp, x29
; VBITS_GE_256-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; VBITS_GE_256-NEXT:    ret
;
; VBITS_GE_512-LABEL: fcvt_v16bf16_v16f32:
; VBITS_GE_512:       // %bb.0:
; VBITS_GE_512-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; VBITS_GE_512-NEXT:    sub x9, sp, #112
; VBITS_GE_512-NEXT:    mov x29, sp
; VBITS_GE_512-NEXT:    and sp, x9, #0xffffffffffffffc0
; VBITS_GE_512-NEXT:    .cfi_def_cfa w29, 16
; VBITS_GE_512-NEXT:    .cfi_offset w30, -8
; VBITS_GE_512-NEXT:    .cfi_offset w29, -16
; VBITS_GE_512-NEXT:    ldr h0, [x0, #30]
; VBITS_GE_512-NEXT:    ptrue p0.s, vl16
; VBITS_GE_512-NEXT:    mov x8, sp
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #60]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #28]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #56]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #26]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #52]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #24]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #48]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #22]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #44]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #20]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #40]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #18]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #36]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #16]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #32]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #14]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #28]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #12]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #24]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #10]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #20]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #8]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #16]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #6]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #12]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #4]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #8]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #2]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp, #4]
; VBITS_GE_512-NEXT:    ldr h0, [x0]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    str s0, [sp]
; VBITS_GE_512-NEXT:    ld1w { z0.s }, p0/z, [x8]
; VBITS_GE_512-NEXT:    st1w { z0.s }, p0, [x1]
; VBITS_GE_512-NEXT:    mov sp, x29
; VBITS_GE_512-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; VBITS_GE_512-NEXT:    ret
;
; VBITS_GE_2048-LABEL: fcvt_v16bf16_v16f32:
; VBITS_GE_2048:       // %bb.0:
; VBITS_GE_2048-NEXT:    ldp q1, q0, [x0]
; VBITS_GE_2048-NEXT:    ptrue p0.s, vl8
; VBITS_GE_2048-NEXT:    uunpklo z0.s, z0.h
; VBITS_GE_2048-NEXT:    uunpklo z1.s, z1.h
; VBITS_GE_2048-NEXT:    lsl z3.s, z0.s, #16
; VBITS_GE_2048-NEXT:    lsl z2.s, z1.s, #16
; VBITS_GE_2048-NEXT:    splice z0.s, p0, { z2.s, z3.s }
; VBITS_GE_2048-NEXT:    ptrue p0.s, vl16
; VBITS_GE_2048-NEXT:    st1w { z0.s }, p0, [x1]
; VBITS_GE_2048-NEXT:    ret
  %op1 = load <16 x bfloat>, ptr %a
  %res = fpext <16 x bfloat> %op1 to <16 x float>
  store <16 x float> %res, ptr %b
  ret void
}

define void @fcvt_v32bf16_v32f32(ptr %a, ptr %b) vscale_range(8,0) {
; CHECK-LABEL: fcvt_v32bf16_v32f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; CHECK-NEXT:    sub x9, sp, #240
; CHECK-NEXT:    mov x29, sp
; CHECK-NEXT:    and sp, x9, #0xffffffffffffff80
; CHECK-NEXT:    .cfi_def_cfa w29, 16
; CHECK-NEXT:    .cfi_offset w30, -8
; CHECK-NEXT:    .cfi_offset w29, -16
; CHECK-NEXT:    ldr h0, [x0, #62]
; CHECK-NEXT:    ptrue p0.s, vl32
; CHECK-NEXT:    mov x8, sp
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #124]
; CHECK-NEXT:    ldr h0, [x0, #60]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #120]
; CHECK-NEXT:    ldr h0, [x0, #58]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #116]
; CHECK-NEXT:    ldr h0, [x0, #56]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #112]
; CHECK-NEXT:    ldr h0, [x0, #54]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #108]
; CHECK-NEXT:    ldr h0, [x0, #52]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #104]
; CHECK-NEXT:    ldr h0, [x0, #50]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #100]
; CHECK-NEXT:    ldr h0, [x0, #48]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #96]
; CHECK-NEXT:    ldr h0, [x0, #46]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #92]
; CHECK-NEXT:    ldr h0, [x0, #44]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #88]
; CHECK-NEXT:    ldr h0, [x0, #42]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #84]
; CHECK-NEXT:    ldr h0, [x0, #40]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #80]
; CHECK-NEXT:    ldr h0, [x0, #38]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #76]
; CHECK-NEXT:    ldr h0, [x0, #36]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #72]
; CHECK-NEXT:    ldr h0, [x0, #34]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #68]
; CHECK-NEXT:    ldr h0, [x0, #32]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #64]
; CHECK-NEXT:    ldr h0, [x0, #30]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #60]
; CHECK-NEXT:    ldr h0, [x0, #28]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #56]
; CHECK-NEXT:    ldr h0, [x0, #26]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #52]
; CHECK-NEXT:    ldr h0, [x0, #24]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #48]
; CHECK-NEXT:    ldr h0, [x0, #22]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #44]
; CHECK-NEXT:    ldr h0, [x0, #20]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #40]
; CHECK-NEXT:    ldr h0, [x0, #18]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #36]
; CHECK-NEXT:    ldr h0, [x0, #16]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #32]
; CHECK-NEXT:    ldr h0, [x0, #14]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #28]
; CHECK-NEXT:    ldr h0, [x0, #12]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #24]
; CHECK-NEXT:    ldr h0, [x0, #10]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #20]
; CHECK-NEXT:    ldr h0, [x0, #8]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #16]
; CHECK-NEXT:    ldr h0, [x0, #6]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #12]
; CHECK-NEXT:    ldr h0, [x0, #4]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #8]
; CHECK-NEXT:    ldr h0, [x0, #2]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #4]
; CHECK-NEXT:    ldr h0, [x0]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp]
; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x8]
; CHECK-NEXT:    st1w { z0.s }, p0, [x1]
; CHECK-NEXT:    mov sp, x29
; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  %op1 = load <32 x bfloat>, ptr %a
  %res = fpext <32 x bfloat> %op1 to <32 x float>
  store <32 x float> %res, ptr %b
  ret void
}

define void @fcvt_v64bf16_v64f32(ptr %a, ptr %b) vscale_range(16,0) {
; CHECK-LABEL: fcvt_v64bf16_v64f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; CHECK-NEXT:    sub x9, sp, #496
; CHECK-NEXT:    mov x29, sp
; CHECK-NEXT:    and sp, x9, #0xffffffffffffff00
; CHECK-NEXT:    .cfi_def_cfa w29, 16
; CHECK-NEXT:    .cfi_offset w30, -8
; CHECK-NEXT:    .cfi_offset w29, -16
; CHECK-NEXT:    ldr h0, [x0, #126]
; CHECK-NEXT:    ptrue p0.s, vl64
; CHECK-NEXT:    mov x8, sp
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #252]
; CHECK-NEXT:    ldr h0, [x0, #124]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #248]
; CHECK-NEXT:    ldr h0, [x0, #122]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #244]
; CHECK-NEXT:    ldr h0, [x0, #120]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #240]
; CHECK-NEXT:    ldr h0, [x0, #118]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #236]
; CHECK-NEXT:    ldr h0, [x0, #116]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #232]
; CHECK-NEXT:    ldr h0, [x0, #114]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #228]
; CHECK-NEXT:    ldr h0, [x0, #112]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #224]
; CHECK-NEXT:    ldr h0, [x0, #110]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #220]
; CHECK-NEXT:    ldr h0, [x0, #108]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #216]
; CHECK-NEXT:    ldr h0, [x0, #106]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #212]
; CHECK-NEXT:    ldr h0, [x0, #104]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #208]
; CHECK-NEXT:    ldr h0, [x0, #102]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #204]
; CHECK-NEXT:    ldr h0, [x0, #100]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #200]
; CHECK-NEXT:    ldr h0, [x0, #98]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #196]
; CHECK-NEXT:    ldr h0, [x0, #96]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #192]
; CHECK-NEXT:    ldr h0, [x0, #94]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #188]
; CHECK-NEXT:    ldr h0, [x0, #92]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #184]
; CHECK-NEXT:    ldr h0, [x0, #90]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #180]
; CHECK-NEXT:    ldr h0, [x0, #88]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #176]
; CHECK-NEXT:    ldr h0, [x0, #86]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #172]
; CHECK-NEXT:    ldr h0, [x0, #84]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #168]
; CHECK-NEXT:    ldr h0, [x0, #82]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #164]
; CHECK-NEXT:    ldr h0, [x0, #80]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #160]
; CHECK-NEXT:    ldr h0, [x0, #78]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #156]
; CHECK-NEXT:    ldr h0, [x0, #76]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #152]
; CHECK-NEXT:    ldr h0, [x0, #74]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #148]
; CHECK-NEXT:    ldr h0, [x0, #72]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #144]
; CHECK-NEXT:    ldr h0, [x0, #70]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #140]
; CHECK-NEXT:    ldr h0, [x0, #68]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #136]
; CHECK-NEXT:    ldr h0, [x0, #66]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #132]
; CHECK-NEXT:    ldr h0, [x0, #64]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #128]
; CHECK-NEXT:    ldr h0, [x0, #62]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #124]
; CHECK-NEXT:    ldr h0, [x0, #60]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #120]
; CHECK-NEXT:    ldr h0, [x0, #58]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #116]
; CHECK-NEXT:    ldr h0, [x0, #56]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #112]
; CHECK-NEXT:    ldr h0, [x0, #54]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #108]
; CHECK-NEXT:    ldr h0, [x0, #52]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #104]
; CHECK-NEXT:    ldr h0, [x0, #50]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #100]
; CHECK-NEXT:    ldr h0, [x0, #48]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #96]
; CHECK-NEXT:    ldr h0, [x0, #46]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #92]
; CHECK-NEXT:    ldr h0, [x0, #44]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #88]
; CHECK-NEXT:    ldr h0, [x0, #42]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #84]
; CHECK-NEXT:    ldr h0, [x0, #40]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #80]
; CHECK-NEXT:    ldr h0, [x0, #38]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #76]
; CHECK-NEXT:    ldr h0, [x0, #36]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #72]
; CHECK-NEXT:    ldr h0, [x0, #34]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #68]
; CHECK-NEXT:    ldr h0, [x0, #32]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #64]
; CHECK-NEXT:    ldr h0, [x0, #30]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #60]
; CHECK-NEXT:    ldr h0, [x0, #28]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #56]
; CHECK-NEXT:    ldr h0, [x0, #26]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #52]
; CHECK-NEXT:    ldr h0, [x0, #24]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #48]
; CHECK-NEXT:    ldr h0, [x0, #22]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #44]
; CHECK-NEXT:    ldr h0, [x0, #20]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #40]
; CHECK-NEXT:    ldr h0, [x0, #18]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #36]
; CHECK-NEXT:    ldr h0, [x0, #16]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #32]
; CHECK-NEXT:    ldr h0, [x0, #14]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #28]
; CHECK-NEXT:    ldr h0, [x0, #12]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #24]
; CHECK-NEXT:    ldr h0, [x0, #10]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #20]
; CHECK-NEXT:    ldr h0, [x0, #8]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #16]
; CHECK-NEXT:    ldr h0, [x0, #6]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #12]
; CHECK-NEXT:    ldr h0, [x0, #4]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #8]
; CHECK-NEXT:    ldr h0, [x0, #2]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp, #4]
; CHECK-NEXT:    ldr h0, [x0]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    str s0, [sp]
; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x8]
; CHECK-NEXT:    st1w { z0.s }, p0, [x1]
; CHECK-NEXT:    mov sp, x29
; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  %op1 = load <64 x bfloat>, ptr %a
  %res = fpext <64 x bfloat> %op1 to <64 x float>
  store <64 x float> %res, ptr %b
  ret void
}


define void @fcvt_v1bf16_v1f64(ptr %a, ptr %b) vscale_range(2,0) {
; CHECK-LABEL: fcvt_v1bf16_v1f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr h0, [x0]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [x1]
; CHECK-NEXT:    ret
  %op1 = load <1 x bfloat>, ptr %a
  %res = fpext <1 x bfloat> %op1 to <1 x double>
  store <1 x double> %res, ptr %b
  ret void
}

define void @fcvt_v2bf16_v2f64(ptr %a, ptr %b) vscale_range(2,0) {
; CHECK-LABEL: fcvt_v2bf16_v2f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr s0, [x0]
; CHECK-NEXT:    ptrue p0.d, vl4
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    uunpklo z0.d, z0.s
; CHECK-NEXT:    fcvt z0.d, p0/m, z0.s
; CHECK-NEXT:    str q0, [x1]
; CHECK-NEXT:    ret
  %op1 = load <2 x bfloat>, ptr %a
  %res = fpext <2 x bfloat> %op1 to <2 x double>
  store <2 x double> %res, ptr %b
  ret void
}

define void @fcvt_v4bf16_v4f64(ptr %a, ptr %b) vscale_range(2,0) {
; CHECK-LABEL: fcvt_v4bf16_v4f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; CHECK-NEXT:    sub x9, sp, #48
; CHECK-NEXT:    mov x29, sp
; CHECK-NEXT:    and sp, x9, #0xffffffffffffffe0
; CHECK-NEXT:    .cfi_def_cfa w29, 16
; CHECK-NEXT:    .cfi_offset w30, -8
; CHECK-NEXT:    .cfi_offset w29, -16
; CHECK-NEXT:    ldr h0, [x0, #6]
; CHECK-NEXT:    ptrue p0.d, vl4
; CHECK-NEXT:    mov x8, sp
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #24]
; CHECK-NEXT:    ldr h0, [x0, #4]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #16]
; CHECK-NEXT:    ldr h0, [x0, #2]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #8]
; CHECK-NEXT:    ldr h0, [x0]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp]
; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8]
; CHECK-NEXT:    st1d { z0.d }, p0, [x1]
; CHECK-NEXT:    mov sp, x29
; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  %op1 = load <4 x bfloat>, ptr %a
  %res = fpext <4 x bfloat> %op1 to <4 x double>
  store <4 x double> %res, ptr %b
  ret void
}

define void @fcvt_v8bf16_v8f64(ptr %a, ptr %b) {
; VBITS_GE_256-LABEL: fcvt_v8bf16_v8f64:
; VBITS_GE_256:       // %bb.0:
; VBITS_GE_256-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; VBITS_GE_256-NEXT:    sub x9, sp, #80
; VBITS_GE_256-NEXT:    mov x29, sp
; VBITS_GE_256-NEXT:    and sp, x9, #0xffffffffffffffe0
; VBITS_GE_256-NEXT:    .cfi_def_cfa w29, 16
; VBITS_GE_256-NEXT:    .cfi_offset w30, -8
; VBITS_GE_256-NEXT:    .cfi_offset w29, -16
; VBITS_GE_256-NEXT:    ldr h0, [x0, #6]
; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
; VBITS_GE_256-NEXT:    add x8, sp, #32
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    fcvt d0, s0
; VBITS_GE_256-NEXT:    str d0, [sp, #24]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #4]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    fcvt d0, s0
; VBITS_GE_256-NEXT:    str d0, [sp, #16]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #2]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    fcvt d0, s0
; VBITS_GE_256-NEXT:    str d0, [sp, #8]
; VBITS_GE_256-NEXT:    ldr h0, [x0]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    fcvt d0, s0
; VBITS_GE_256-NEXT:    str d0, [sp]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #14]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    fcvt d0, s0
; VBITS_GE_256-NEXT:    str d0, [sp, #56]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #12]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    fcvt d0, s0
; VBITS_GE_256-NEXT:    str d0, [sp, #48]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #10]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    fcvt d0, s0
; VBITS_GE_256-NEXT:    str d0, [sp, #40]
; VBITS_GE_256-NEXT:    ldr h0, [x0, #8]
; VBITS_GE_256-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_256-NEXT:    fcvt d0, s0
; VBITS_GE_256-NEXT:    str d0, [sp, #32]
; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x8]
; VBITS_GE_256-NEXT:    mov x8, sp
; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x8]
; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1]
; VBITS_GE_256-NEXT:    mov sp, x29
; VBITS_GE_256-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; VBITS_GE_256-NEXT:    ret
;
; VBITS_GE_512-LABEL: fcvt_v8bf16_v8f64:
; VBITS_GE_512:       // %bb.0:
; VBITS_GE_512-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; VBITS_GE_512-NEXT:    sub x9, sp, #112
; VBITS_GE_512-NEXT:    mov x29, sp
; VBITS_GE_512-NEXT:    and sp, x9, #0xffffffffffffffc0
; VBITS_GE_512-NEXT:    .cfi_def_cfa w29, 16
; VBITS_GE_512-NEXT:    .cfi_offset w30, -8
; VBITS_GE_512-NEXT:    .cfi_offset w29, -16
; VBITS_GE_512-NEXT:    ldr h0, [x0, #14]
; VBITS_GE_512-NEXT:    ptrue p0.d, vl8
; VBITS_GE_512-NEXT:    mov x8, sp
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    fcvt d0, s0
; VBITS_GE_512-NEXT:    str d0, [sp, #56]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #12]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    fcvt d0, s0
; VBITS_GE_512-NEXT:    str d0, [sp, #48]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #10]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    fcvt d0, s0
; VBITS_GE_512-NEXT:    str d0, [sp, #40]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #8]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    fcvt d0, s0
; VBITS_GE_512-NEXT:    str d0, [sp, #32]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #6]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    fcvt d0, s0
; VBITS_GE_512-NEXT:    str d0, [sp, #24]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #4]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    fcvt d0, s0
; VBITS_GE_512-NEXT:    str d0, [sp, #16]
; VBITS_GE_512-NEXT:    ldr h0, [x0, #2]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    fcvt d0, s0
; VBITS_GE_512-NEXT:    str d0, [sp, #8]
; VBITS_GE_512-NEXT:    ldr h0, [x0]
; VBITS_GE_512-NEXT:    shll v0.4s, v0.4h, #16
; VBITS_GE_512-NEXT:    fcvt d0, s0
; VBITS_GE_512-NEXT:    str d0, [sp]
; VBITS_GE_512-NEXT:    ld1d { z0.d }, p0/z, [x8]
; VBITS_GE_512-NEXT:    st1d { z0.d }, p0, [x1]
; VBITS_GE_512-NEXT:    mov sp, x29
; VBITS_GE_512-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; VBITS_GE_512-NEXT:    ret
;
; VBITS_GE_2048-LABEL: fcvt_v8bf16_v8f64:
; VBITS_GE_2048:       // %bb.0:
; VBITS_GE_2048-NEXT:    ldr q0, [x0]
; VBITS_GE_2048-NEXT:    ptrue p0.d, vl8
; VBITS_GE_2048-NEXT:    uunpklo z0.s, z0.h
; VBITS_GE_2048-NEXT:    lsl z0.s, z0.s, #16
; VBITS_GE_2048-NEXT:    uunpklo z0.d, z0.s
; VBITS_GE_2048-NEXT:    fcvt z0.d, p0/m, z0.s
; VBITS_GE_2048-NEXT:    st1d { z0.d }, p0, [x1]
; VBITS_GE_2048-NEXT:    ret
  %op1 = load <8 x bfloat>, ptr %a
  %res = fpext <8 x bfloat> %op1 to <8 x double>
  store <8 x double> %res, ptr %b
  ret void
}

define void @fcvt_v16bf16_v16f64(ptr %a, ptr %b) vscale_range(8,0) {
; CHECK-LABEL: fcvt_v16bf16_v16f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; CHECK-NEXT:    sub x9, sp, #240
; CHECK-NEXT:    mov x29, sp
; CHECK-NEXT:    and sp, x9, #0xffffffffffffff80
; CHECK-NEXT:    .cfi_def_cfa w29, 16
; CHECK-NEXT:    .cfi_offset w30, -8
; CHECK-NEXT:    .cfi_offset w29, -16
; CHECK-NEXT:    ldr h0, [x0, #30]
; CHECK-NEXT:    ptrue p0.d, vl16
; CHECK-NEXT:    mov x8, sp
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #120]
; CHECK-NEXT:    ldr h0, [x0, #28]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #112]
; CHECK-NEXT:    ldr h0, [x0, #26]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #104]
; CHECK-NEXT:    ldr h0, [x0, #24]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #96]
; CHECK-NEXT:    ldr h0, [x0, #22]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #88]
; CHECK-NEXT:    ldr h0, [x0, #20]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #80]
; CHECK-NEXT:    ldr h0, [x0, #18]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #72]
; CHECK-NEXT:    ldr h0, [x0, #16]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #64]
; CHECK-NEXT:    ldr h0, [x0, #14]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #56]
; CHECK-NEXT:    ldr h0, [x0, #12]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #48]
; CHECK-NEXT:    ldr h0, [x0, #10]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #40]
; CHECK-NEXT:    ldr h0, [x0, #8]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #32]
; CHECK-NEXT:    ldr h0, [x0, #6]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #24]
; CHECK-NEXT:    ldr h0, [x0, #4]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #16]
; CHECK-NEXT:    ldr h0, [x0, #2]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #8]
; CHECK-NEXT:    ldr h0, [x0]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp]
; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8]
; CHECK-NEXT:    st1d { z0.d }, p0, [x1]
; CHECK-NEXT:    mov sp, x29
; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  %op1 = load <16 x bfloat>, ptr %a
  %res = fpext <16 x bfloat> %op1 to <16 x double>
  store <16 x double> %res, ptr %b
  ret void
}

define void @fcvt_v32bf16_v32f64(ptr %a, ptr %b) vscale_range(16,0) {
; CHECK-LABEL: fcvt_v32bf16_v32f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; CHECK-NEXT:    sub x9, sp, #496
; CHECK-NEXT:    mov x29, sp
; CHECK-NEXT:    and sp, x9, #0xffffffffffffff00
; CHECK-NEXT:    .cfi_def_cfa w29, 16
; CHECK-NEXT:    .cfi_offset w30, -8
; CHECK-NEXT:    .cfi_offset w29, -16
; CHECK-NEXT:    ldr h0, [x0, #62]
; CHECK-NEXT:    ptrue p0.d, vl32
; CHECK-NEXT:    mov x8, sp
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #248]
; CHECK-NEXT:    ldr h0, [x0, #60]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #240]
; CHECK-NEXT:    ldr h0, [x0, #58]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #232]
; CHECK-NEXT:    ldr h0, [x0, #56]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #224]
; CHECK-NEXT:    ldr h0, [x0, #54]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #216]
; CHECK-NEXT:    ldr h0, [x0, #52]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #208]
; CHECK-NEXT:    ldr h0, [x0, #50]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #200]
; CHECK-NEXT:    ldr h0, [x0, #48]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #192]
; CHECK-NEXT:    ldr h0, [x0, #46]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #184]
; CHECK-NEXT:    ldr h0, [x0, #44]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #176]
; CHECK-NEXT:    ldr h0, [x0, #42]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #168]
; CHECK-NEXT:    ldr h0, [x0, #40]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #160]
; CHECK-NEXT:    ldr h0, [x0, #38]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #152]
; CHECK-NEXT:    ldr h0, [x0, #36]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #144]
; CHECK-NEXT:    ldr h0, [x0, #34]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #136]
; CHECK-NEXT:    ldr h0, [x0, #32]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #128]
; CHECK-NEXT:    ldr h0, [x0, #30]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #120]
; CHECK-NEXT:    ldr h0, [x0, #28]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #112]
; CHECK-NEXT:    ldr h0, [x0, #26]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #104]
; CHECK-NEXT:    ldr h0, [x0, #24]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #96]
; CHECK-NEXT:    ldr h0, [x0, #22]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #88]
; CHECK-NEXT:    ldr h0, [x0, #20]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #80]
; CHECK-NEXT:    ldr h0, [x0, #18]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #72]
; CHECK-NEXT:    ldr h0, [x0, #16]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #64]
; CHECK-NEXT:    ldr h0, [x0, #14]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #56]
; CHECK-NEXT:    ldr h0, [x0, #12]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #48]
; CHECK-NEXT:    ldr h0, [x0, #10]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #40]
; CHECK-NEXT:    ldr h0, [x0, #8]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #32]
; CHECK-NEXT:    ldr h0, [x0, #6]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #24]
; CHECK-NEXT:    ldr h0, [x0, #4]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #16]
; CHECK-NEXT:    ldr h0, [x0, #2]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp, #8]
; CHECK-NEXT:    ldr h0, [x0]
; CHECK-NEXT:    shll v0.4s, v0.4h, #16
; CHECK-NEXT:    fcvt d0, s0
; CHECK-NEXT:    str d0, [sp]
; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x8]
; CHECK-NEXT:    st1d { z0.d }, p0, [x1]
; CHECK-NEXT:    mov sp, x29
; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  %op1 = load <32 x bfloat>, ptr %a
  %res = fpext <32 x bfloat> %op1 to <32 x double>
  store <32 x double> %res, ptr %b
  ret void
}