; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=+avx -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,AVX
; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=+avx2 -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,AVX2
; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=+avx512bf16,+avx512vl -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,AVX512BF16
; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=+avx512f -O0 -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,AVX512F-O0

define <4 x bfloat> @fptrunc_v4f32(<4 x float> %x) nounwind {
; AVX-LABEL: fptrunc_v4f32:
; AVX:       # %bb.0:
; AVX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX-NEXT:    vcmpunordps %ymm0, %ymm0, %ymm1
; AVX-NEXT:    vpsrld $16, %xmm0, %xmm2
; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
; AVX-NEXT:    vpaddd %xmm0, %xmm2, %xmm2
; AVX-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 # [32767,32767,32767,32767]
; AVX-NEXT:    vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVX-NEXT:    vblendvps %xmm1, %xmm0, %xmm2, %xmm0
; AVX-NEXT:    vpsrld $16, %xmm0, %xmm0
; AVX-NEXT:    vpackusdw %xmm0, %xmm0, %xmm1
; AVX-NEXT:    vmovq %xmm1, %rax
; AVX-NEXT:    movl %eax, %ecx
; AVX-NEXT:    shrl $16, %ecx
; AVX-NEXT:    movq %rax, %rdx
; AVX-NEXT:    shrq $32, %rdx
; AVX-NEXT:    shrq $48, %rax
; AVX-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm1
; AVX-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm2
; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm2
; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: fptrunc_v4f32:
; AVX2:       # %bb.0:
; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4194304,4194304,4194304,4194304]
; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm1
; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm2
; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [1,1,1,1]
; AVX2-NEXT:    vpand %xmm3, %xmm2, %xmm2
; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [32767,32767,32767,32767]
; AVX2-NEXT:    vpaddd %xmm3, %xmm0, %xmm3
; AVX2-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
; AVX2-NEXT:    vcmpunordps %ymm0, %ymm0, %ymm0
; AVX2-NEXT:    vblendvps %xmm0, %xmm1, %xmm2, %xmm0
; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm0
; AVX2-NEXT:    vpackusdw %xmm0, %xmm0, %xmm1
; AVX2-NEXT:    vmovq %xmm1, %rax
; AVX2-NEXT:    movl %eax, %ecx
; AVX2-NEXT:    shrl $16, %ecx
; AVX2-NEXT:    movq %rax, %rdx
; AVX2-NEXT:    shrq $32, %rdx
; AVX2-NEXT:    shrq $48, %rax
; AVX2-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm1
; AVX2-NEXT:    vpinsrw $0, %edx, %xmm0, %xmm2
; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX2-NEXT:    vpinsrw $0, %ecx, %xmm0, %xmm2
; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512BF16-LABEL: fptrunc_v4f32:
; AVX512BF16:       # %bb.0:
; AVX512BF16-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512BF16-NEXT:    vcvtneps2bf16 %ymm0, %xmm0
; AVX512BF16-NEXT:    vzeroupper
; AVX512BF16-NEXT:    retq
;
; AVX512F-O0-LABEL: fptrunc_v4f32:
; AVX512F-O0:       # %bb.0:
; AVX512F-O0-NEXT:    # implicit-def: $ymm2
; AVX512F-O0-NEXT:    vmovaps %xmm0, %xmm2
; AVX512F-O0-NEXT:    vpsrld $16, %ymm2, %ymm0
; AVX512F-O0-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [1,1,1,1,1,1,1,1]
; AVX512F-O0-NEXT:    vpand %ymm1, %ymm0, %ymm0
; AVX512F-O0-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
; AVX512F-O0-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
; AVX512F-O0-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
; AVX512F-O0-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4194304,4194304,4194304,4194304,4194304,4194304,4194304,4194304]
; AVX512F-O0-NEXT:    vpor %ymm1, %ymm2, %ymm1
; AVX512F-O0-NEXT:    vcmpunordps %ymm2, %ymm2, %ymm2
; AVX512F-O0-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
; AVX512F-O0-NEXT:    vpsrld $16, %ymm0, %ymm1
; AVX512F-O0-NEXT:    # implicit-def: $zmm0
; AVX512F-O0-NEXT:    vmovaps %ymm1, %ymm0
; AVX512F-O0-NEXT:    vpmovdw %zmm0, %ymm0
; AVX512F-O0-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
; AVX512F-O0-NEXT:    vzeroupper
; AVX512F-O0-NEXT:    retq
  %r = fptrunc <4 x float> %x to <4 x bfloat>
  ret <4 x bfloat> %r
}

define <8 x bfloat> @fptrunc_v8f32(<8 x float> %x) nounwind {
; AVX-LABEL: fptrunc_v8f32:
; AVX:       # %bb.0:
; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
; AVX-NEXT:    vpsrld $16, %xmm1, %xmm2
; AVX-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
; AVX-NEXT:    vpand %xmm3, %xmm2, %xmm2
; AVX-NEXT:    vbroadcastss {{.*#+}} xmm4 = [32767,32767,32767,32767]
; AVX-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
; AVX-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
; AVX-NEXT:    vpsrld $16, %xmm0, %xmm2
; AVX-NEXT:    vpand %xmm3, %xmm2, %xmm2
; AVX-NEXT:    vpaddd %xmm4, %xmm0, %xmm3
; AVX-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX-NEXT:    vcmpunordps %ymm0, %ymm0, %ymm2
; AVX-NEXT:    vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
; AVX-NEXT:    vpsrld $16, %xmm1, %xmm1
; AVX-NEXT:    vpsrld $16, %xmm0, %xmm0
; AVX-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: fptrunc_v8f32:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4194304,4194304,4194304,4194304,4194304,4194304,4194304,4194304]
; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm1
; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm2
; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1]
; AVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [32767,32767,32767,32767,32767,32767,32767,32767]
; AVX2-NEXT:    vpaddd %ymm3, %ymm0, %ymm3
; AVX2-NEXT:    vpaddd %ymm3, %ymm2, %ymm2
; AVX2-NEXT:    vcmpunordps %ymm0, %ymm0, %ymm0
; AVX2-NEXT:    vblendvps %ymm0, %ymm1, %ymm2, %ymm0
; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512BF16-LABEL: fptrunc_v8f32:
; AVX512BF16:       # %bb.0:
; AVX512BF16-NEXT:    vcvtneps2bf16 %ymm0, %xmm0
; AVX512BF16-NEXT:    vzeroupper
; AVX512BF16-NEXT:    retq
;
; AVX512F-O0-LABEL: fptrunc_v8f32:
; AVX512F-O0:       # %bb.0:
; AVX512F-O0-NEXT:    vmovaps %ymm0, %ymm2
; AVX512F-O0-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [4194304,4194304,4194304,4194304,4194304,4194304,4194304,4194304]
; AVX512F-O0-NEXT:    vpor %ymm0, %ymm2, %ymm1
; AVX512F-O0-NEXT:    vpsrld $16, %ymm2, %ymm0
; AVX512F-O0-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [1,1,1,1,1,1,1,1]
; AVX512F-O0-NEXT:    vpand %ymm3, %ymm0, %ymm0
; AVX512F-O0-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
; AVX512F-O0-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [32767,32767,32767,32767,32767,32767,32767,32767]
; AVX512F-O0-NEXT:    vpaddd %ymm3, %ymm0, %ymm0
; AVX512F-O0-NEXT:    vcmpunordps %ymm2, %ymm2, %ymm2
; AVX512F-O0-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
; AVX512F-O0-NEXT:    vpsrld $16, %ymm0, %ymm1
; AVX512F-O0-NEXT:    # implicit-def: $zmm0
; AVX512F-O0-NEXT:    vmovaps %ymm1, %ymm0
; AVX512F-O0-NEXT:    vpmovdw %zmm0, %ymm0
; AVX512F-O0-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
; AVX512F-O0-NEXT:    vzeroupper
; AVX512F-O0-NEXT:    retq
  %r = fptrunc <8 x float> %x to <8 x bfloat>
  ret <8 x bfloat> %r
}

; Keep basic operations on the newly legal v8bf16 type from reaching the
; unsupported soft-promotion path.

define <8 x bfloat> @insert_v8bf16(<8 x bfloat> %x, bfloat %y, i64 %idx) {
; AVX-LABEL: insert_v8bf16:
; AVX:       # %bb.0:
; AVX-NEXT:    andl $7, %edi
; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
; AVX-NEXT:    vpextrw $0, %xmm1, -24(%rsp,%rdi,2)
; AVX-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX-NEXT:    retq
;
; AVX2-LABEL: insert_v8bf16:
; AVX2:       # %bb.0:
; AVX2-NEXT:    andl $7, %edi
; AVX2-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT:    vpextrw $0, %xmm1, -24(%rsp,%rdi,2)
; AVX2-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT:    retq
;
; AVX512BF16-LABEL: insert_v8bf16:
; AVX512BF16:       # %bb.0:
; AVX512BF16-NEXT:    vpbroadcastw %edi, %xmm2
; AVX512BF16-NEXT:    vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1
; AVX512BF16-NEXT:    vpbroadcastw %xmm1, %xmm1
; AVX512BF16-NEXT:    vmovdqu16 %xmm1, %xmm0 {%k1}
; AVX512BF16-NEXT:    retq
;
; AVX512F-O0-LABEL: insert_v8bf16:
; AVX512F-O0:       # %bb.0:
; AVX512F-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; AVX512F-O0-NEXT:    vmovaps %xmm0, %xmm1
; AVX512F-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; AVX512F-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; AVX512F-O0-NEXT:    vpslld $16, %xmm0, %xmm0
; AVX512F-O0-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
; AVX512F-O0-NEXT:    vmovd %xmm0, %eax
; AVX512F-O0-NEXT:    shrl $16, %eax
; AVX512F-O0-NEXT:    movw %ax, %cx
; AVX512F-O0-NEXT:    movl %edi, %eax
; AVX512F-O0-NEXT:    andl $7, %eax
; AVX512F-O0-NEXT:    # kill: def $rax killed $eax
; AVX512F-O0-NEXT:    movw %cx, -24(%rsp,%rax,2)
; AVX512F-O0-NEXT:    vmovdqa -{{[0-9]+}}(%rsp), %xmm0
; AVX512F-O0-NEXT:    retq
  %r = insertelement <8 x bfloat> %x, bfloat %y, i64 %idx
  ret <8 x bfloat> %r
}

define <8 x bfloat> @build_v8bf16(bfloat %x, bfloat %y) {
; AVX-LABEL: build_v8bf16:
; AVX:       # %bb.0:
; AVX-NEXT:    vpextrw $0, %xmm1, %eax
; AVX-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0
; AVX-NEXT:    retq
;
; AVX2-LABEL: build_v8bf16:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpextrw $0, %xmm1, %eax
; AVX2-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0
; AVX2-NEXT:    retq
;
; AVX512BF16-LABEL: build_v8bf16:
; AVX512BF16:       # %bb.0:
; AVX512BF16-NEXT:    vpextrw $0, %xmm1, %eax
; AVX512BF16-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0
; AVX512BF16-NEXT:    retq
;
; AVX512F-O0-LABEL: build_v8bf16:
; AVX512F-O0:       # %bb.0:
; AVX512F-O0-NEXT:    vpslld $16, %xmm1, %xmm1
; AVX512F-O0-NEXT:    vpslld $16, %xmm0, %xmm0
; AVX512F-O0-NEXT:    vpsrld $16, %xmm0, %xmm0
; AVX512F-O0-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
; AVX512F-O0-NEXT:    retq
  %r0 = insertelement <8 x bfloat> poison, bfloat %x, i64 0
  %r1 = insertelement <8 x bfloat> %r0, bfloat %y, i64 1
  ret <8 x bfloat> %r1
}

define <8 x bfloat> @canonicalize_v8bf16(<8 x bfloat> %x) {
; AVX-LABEL: canonicalize_v8bf16:
; AVX:       # %bb.0:
; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
; AVX-NEXT:    vpsrld $16, %xmm1, %xmm2
; AVX-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,1,1,1]
; AVX-NEXT:    vpand %xmm3, %xmm2, %xmm2
; AVX-NEXT:    vbroadcastss {{.*#+}} xmm4 = [32767,32767,32767,32767]
; AVX-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
; AVX-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
; AVX-NEXT:    vpsrld $16, %xmm0, %xmm2
; AVX-NEXT:    vpand %xmm3, %xmm2, %xmm2
; AVX-NEXT:    vpaddd %xmm4, %xmm0, %xmm3
; AVX-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX-NEXT:    vcmpunordps %ymm0, %ymm0, %ymm2
; AVX-NEXT:    vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; AVX-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
; AVX-NEXT:    vpsrld $16, %xmm1, %xmm1
; AVX-NEXT:    vpsrld $16, %xmm0, %xmm0
; AVX-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: canonicalize_v8bf16:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
; AVX2-NEXT:    vmulps %ymm1, %ymm0, %ymm0
; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm1
; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1]
; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [32767,32767,32767,32767,32767,32767,32767,32767]
; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm2
; AVX2-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [4194304,4194304,4194304,4194304,4194304,4194304,4194304,4194304]
; AVX2-NEXT:    vorps %ymm2, %ymm0, %ymm2
; AVX2-NEXT:    vcmpunordps %ymm0, %ymm0, %ymm0
; AVX2-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512BF16-LABEL: canonicalize_v8bf16:
; AVX512BF16:       # %bb.0:
; AVX512BF16-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVX512BF16-NEXT:    vpslld $16, %ymm0, %ymm0
; AVX512BF16-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
; AVX512BF16-NEXT:    vcvtneps2bf16 %ymm0, %xmm0
; AVX512BF16-NEXT:    vzeroupper
; AVX512BF16-NEXT:    retq
;
; AVX512F-O0-LABEL: canonicalize_v8bf16:
; AVX512F-O0:       # %bb.0:
; AVX512F-O0-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVX512F-O0-NEXT:    vpslld $16, %ymm0, %ymm0
; AVX512F-O0-NEXT:    vbroadcastss {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
; AVX512F-O0-NEXT:    vmulps %ymm1, %ymm0, %ymm2
; AVX512F-O0-NEXT:    vpsrld $16, %ymm2, %ymm0
; AVX512F-O0-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [1,1,1,1,1,1,1,1]
; AVX512F-O0-NEXT:    vpand %ymm1, %ymm0, %ymm0
; AVX512F-O0-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
; AVX512F-O0-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
; AVX512F-O0-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
; AVX512F-O0-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4194304,4194304,4194304,4194304,4194304,4194304,4194304,4194304]
; AVX512F-O0-NEXT:    vpor %ymm1, %ymm2, %ymm1
; AVX512F-O0-NEXT:    vcmpunordps %ymm2, %ymm2, %ymm2
; AVX512F-O0-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
; AVX512F-O0-NEXT:    vpsrld $16, %ymm0, %ymm1
; AVX512F-O0-NEXT:    # implicit-def: $zmm0
; AVX512F-O0-NEXT:    vmovaps %ymm1, %ymm0
; AVX512F-O0-NEXT:    vpmovdw %zmm0, %ymm0
; AVX512F-O0-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
; AVX512F-O0-NEXT:    vzeroupper
; AVX512F-O0-NEXT:    retq
  %r = call <8 x bfloat> @llvm.canonicalize.v8bf16(<8 x bfloat> %x)
  ret <8 x bfloat> %r
}

define <16 x i32> @fptosi_v16bf16(<16 x bfloat> %x) nounwind {
; AVX-LABEL: fptosi_v16bf16:
; AVX:       # %bb.0:
; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
; AVX-NEXT:    vcvttps2dq %ymm0, %ymm0
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
; AVX-NEXT:    vcvttps2dq %ymm1, %ymm1
; AVX-NEXT:    retq
;
; AVX2-LABEL: fptosi_v16bf16:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
; AVX2-NEXT:    vcvttps2dq %ymm0, %ymm0
; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
; AVX2-NEXT:    vcvttps2dq %ymm1, %ymm1
; AVX2-NEXT:    retq
;
; AVX512BF16-LABEL: fptosi_v16bf16:
; AVX512BF16:       # %bb.0:
; AVX512BF16-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; AVX512BF16-NEXT:    vpslld $16, %zmm0, %zmm0
; AVX512BF16-NEXT:    vcvttps2dq %zmm0, %zmm0
; AVX512BF16-NEXT:    retq
;
; AVX512F-O0-LABEL: fptosi_v16bf16:
; AVX512F-O0:       # %bb.0:
; AVX512F-O0-NEXT:    vextracti128 $1, %ymm0, %xmm1
; AVX512F-O0-NEXT:    vmovaps %xmm0, %xmm2
; AVX512F-O0-NEXT:    # implicit-def: $ymm0
; AVX512F-O0-NEXT:    vmovaps %xmm2, %xmm0
; AVX512F-O0-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX512F-O0-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; AVX512F-O0-NEXT:    vpslld $16, %zmm0, %zmm0
; AVX512F-O0-NEXT:    vcvttps2dq %zmm0, %zmm0
; AVX512F-O0-NEXT:    retq
  %r = fptosi <16 x bfloat> %x to <16 x i32>
  ret <16 x i32> %r
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK: {{.*}}