| ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 |
| ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64 | FileCheck %s --check-prefixes=SSE |
| ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE |
| ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=AVX2 |
| ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512 |
| ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512bf16 | FileCheck %s --check-prefixes=AVX512BF16,AVX512BF16-NOVLX |
| ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,+avx512bf16 | FileCheck %s --check-prefixes=AVX512BF16,AVX512BF16-VLX |
| ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx10.2 | FileCheck %s --check-prefixes=AVX10,AVX10-512 |
| ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx10.2,+prefer-256-bit | FileCheck %s --check-prefixes=AVX10,AVX10-256 |
| |
| ; All functions carry "min-legal-vector-width"="256" so that on prefer-256-bit |
| ; targets (x86-64-v4, and the +prefer-256-bit RUN line) the 512-bit types are |
| ; legalized as two 256-bit halves instead of being kept in zmm registers. |
| |
| ; bf16 x bf16 -> f32: vdpbf16ps (AVX512-BF16) |
| |
| define <4 x float> @partial_reduce_fmla_bf16_v4f32(<4 x float> %acc, <8 x bfloat> %a, <8 x bfloat> %b) #0 { |
| ; SSE-LABEL: partial_reduce_fmla_bf16_v4f32: |
| ; SSE: # %bb.0: |
| ; SSE-NEXT: pxor %xmm3, %xmm3 |
| ; SSE-NEXT: pxor %xmm4, %xmm4 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3] |
| ; SSE-NEXT: pxor %xmm5, %xmm5 |
| ; SSE-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7] |
| ; SSE-NEXT: pxor %xmm1, %xmm1 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] |
| ; SSE-NEXT: mulps %xmm4, %xmm1 |
| ; SSE-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] |
| ; SSE-NEXT: mulps %xmm5, %xmm3 |
| ; SSE-NEXT: addps %xmm1, %xmm0 |
| ; SSE-NEXT: addps %xmm3, %xmm0 |
| ; SSE-NEXT: retq |
| ; |
| ; AVX2-LABEL: partial_reduce_fmla_bf16_v4f32: |
| ; AVX2: # %bb.0: |
| ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero |
| ; AVX2-NEXT: vpslld $16, %ymm1, %ymm1 |
| ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero |
| ; AVX2-NEXT: vpslld $16, %ymm2, %ymm2 |
| ; AVX2-NEXT: vmulps %ymm2, %ymm1, %ymm1 |
| ; AVX2-NEXT: vaddps %xmm1, %xmm0, %xmm0 |
| ; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm1 |
| ; AVX2-NEXT: vaddps %xmm0, %xmm1, %xmm0 |
| ; AVX2-NEXT: vzeroupper |
| ; AVX2-NEXT: retq |
| ; |
| ; AVX512-LABEL: partial_reduce_fmla_bf16_v4f32: |
| ; AVX512: # %bb.0: |
| ; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero |
| ; AVX512-NEXT: vpslld $16, %ymm1, %ymm1 |
| ; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero |
| ; AVX512-NEXT: vpslld $16, %ymm2, %ymm2 |
| ; AVX512-NEXT: vmulps %ymm2, %ymm1, %ymm1 |
| ; AVX512-NEXT: vaddps %xmm1, %xmm0, %xmm0 |
| ; AVX512-NEXT: vextractf128 $1, %ymm1, %xmm1 |
| ; AVX512-NEXT: vaddps %xmm0, %xmm1, %xmm0 |
| ; AVX512-NEXT: vzeroupper |
| ; AVX512-NEXT: retq |
| ; |
| ; AVX512BF16-NOVLX-LABEL: partial_reduce_fmla_bf16_v4f32: |
| ; AVX512BF16-NOVLX: # %bb.0: |
| ; AVX512BF16-NOVLX-NEXT: # kill: def $xmm2 killed $xmm2 def $zmm2 |
| ; AVX512BF16-NOVLX-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 |
| ; AVX512BF16-NOVLX-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 |
| ; AVX512BF16-NOVLX-NEXT: vdpbf16ps %zmm2, %zmm1, %zmm0 |
| ; AVX512BF16-NOVLX-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 |
| ; AVX512BF16-NOVLX-NEXT: vzeroupper |
| ; AVX512BF16-NOVLX-NEXT: retq |
| ; |
| ; AVX512BF16-VLX-LABEL: partial_reduce_fmla_bf16_v4f32: |
| ; AVX512BF16-VLX: # %bb.0: |
| ; AVX512BF16-VLX-NEXT: vdpbf16ps %xmm2, %xmm1, %xmm0 |
| ; AVX512BF16-VLX-NEXT: retq |
| ; |
| ; AVX10-LABEL: partial_reduce_fmla_bf16_v4f32: |
| ; AVX10: # %bb.0: |
| ; AVX10-NEXT: vdpbf16ps %xmm2, %xmm1, %xmm0 |
| ; AVX10-NEXT: retq |
| %a.ext = fpext <8 x bfloat> %a to <8 x float> |
| %b.ext = fpext <8 x bfloat> %b to <8 x float> |
| %mul = fmul <8 x float> %a.ext, %b.ext |
| %res = call <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> %acc, <8 x float> %mul) |
| ret <4 x float> %res |
| } |
| |
| define <8 x float> @partial_reduce_fmla_bf16_v8f32(<8 x float> %acc, <16 x bfloat> %a, <16 x bfloat> %b) #0 { |
| ; SSE-LABEL: partial_reduce_fmla_bf16_v8f32: |
| ; SSE: # %bb.0: |
| ; SSE-NEXT: pxor %xmm6, %xmm6 |
| ; SSE-NEXT: pxor %xmm7, %xmm7 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3] |
| ; SSE-NEXT: pxor %xmm8, %xmm8 |
| ; SSE-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7] |
| ; SSE-NEXT: pxor %xmm2, %xmm2 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3] |
| ; SSE-NEXT: pxor %xmm9, %xmm9 |
| ; SSE-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm3[4],xmm9[5],xmm3[5],xmm9[6],xmm3[6],xmm9[7],xmm3[7] |
| ; SSE-NEXT: pxor %xmm3, %xmm3 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3] |
| ; SSE-NEXT: mulps %xmm7, %xmm3 |
| ; SSE-NEXT: pxor %xmm7, %xmm7 |
| ; SSE-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7] |
| ; SSE-NEXT: mulps %xmm8, %xmm7 |
| ; SSE-NEXT: pxor %xmm4, %xmm4 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3] |
| ; SSE-NEXT: mulps %xmm2, %xmm4 |
| ; SSE-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7] |
| ; SSE-NEXT: mulps %xmm9, %xmm6 |
| ; SSE-NEXT: addps %xmm3, %xmm0 |
| ; SSE-NEXT: addps %xmm7, %xmm0 |
| ; SSE-NEXT: addps %xmm4, %xmm1 |
| ; SSE-NEXT: addps %xmm6, %xmm1 |
| ; SSE-NEXT: retq |
| ; |
| ; AVX2-LABEL: partial_reduce_fmla_bf16_v8f32: |
| ; AVX2: # %bb.0: |
| ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero |
| ; AVX2-NEXT: vpslld $16, %ymm3, %ymm3 |
| ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1 |
| ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero |
| ; AVX2-NEXT: vpslld $16, %ymm1, %ymm1 |
| ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero |
| ; AVX2-NEXT: vpslld $16, %ymm4, %ymm4 |
| ; AVX2-NEXT: vmulps %ymm4, %ymm3, %ymm3 |
| ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm2 |
| ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero |
| ; AVX2-NEXT: vpslld $16, %ymm2, %ymm2 |
| ; AVX2-NEXT: vmulps %ymm2, %ymm1, %ymm1 |
| ; AVX2-NEXT: vaddps %ymm3, %ymm0, %ymm0 |
| ; AVX2-NEXT: vaddps %ymm1, %ymm0, %ymm0 |
| ; AVX2-NEXT: retq |
| ; |
| ; AVX512-LABEL: partial_reduce_fmla_bf16_v8f32: |
| ; AVX512: # %bb.0: |
| ; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero |
| ; AVX512-NEXT: vpslld $16, %ymm3, %ymm3 |
| ; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1 |
| ; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero |
| ; AVX512-NEXT: vpslld $16, %ymm1, %ymm1 |
| ; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero |
| ; AVX512-NEXT: vpslld $16, %ymm4, %ymm4 |
| ; AVX512-NEXT: vmulps %ymm4, %ymm3, %ymm3 |
| ; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm2 |
| ; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero |
| ; AVX512-NEXT: vpslld $16, %ymm2, %ymm2 |
| ; AVX512-NEXT: vmulps %ymm2, %ymm1, %ymm1 |
| ; AVX512-NEXT: vaddps %ymm3, %ymm0, %ymm0 |
| ; AVX512-NEXT: vaddps %ymm1, %ymm0, %ymm0 |
| ; AVX512-NEXT: retq |
| ; |
| ; AVX512BF16-NOVLX-LABEL: partial_reduce_fmla_bf16_v8f32: |
| ; AVX512BF16-NOVLX: # %bb.0: |
| ; AVX512BF16-NOVLX-NEXT: # kill: def $ymm2 killed $ymm2 def $zmm2 |
| ; AVX512BF16-NOVLX-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 |
| ; AVX512BF16-NOVLX-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 |
| ; AVX512BF16-NOVLX-NEXT: vdpbf16ps %zmm2, %zmm1, %zmm0 |
| ; AVX512BF16-NOVLX-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 |
| ; AVX512BF16-NOVLX-NEXT: retq |
| ; |
| ; AVX512BF16-VLX-LABEL: partial_reduce_fmla_bf16_v8f32: |
| ; AVX512BF16-VLX: # %bb.0: |
| ; AVX512BF16-VLX-NEXT: vdpbf16ps %ymm2, %ymm1, %ymm0 |
| ; AVX512BF16-VLX-NEXT: retq |
| ; |
| ; AVX10-LABEL: partial_reduce_fmla_bf16_v8f32: |
| ; AVX10: # %bb.0: |
| ; AVX10-NEXT: vdpbf16ps %ymm2, %ymm1, %ymm0 |
| ; AVX10-NEXT: retq |
| %a.ext = fpext <16 x bfloat> %a to <16 x float> |
| %b.ext = fpext <16 x bfloat> %b to <16 x float> |
| %mul = fmul <16 x float> %a.ext, %b.ext |
| %res = call <8 x float> @llvm.vector.partial.reduce.fadd.v8f32.v16f32(<8 x float> %acc, <16 x float> %mul) |
| ret <8 x float> %res |
| } |
| |
| define <16 x float> @partial_reduce_fmla_bf16_v16f32(<16 x float> %acc, <32 x bfloat> %a, <32 x bfloat> %b) #0 { |
| ; SSE-LABEL: partial_reduce_fmla_bf16_v16f32: |
| ; SSE: # %bb.0: |
| ; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm9 |
| ; SSE-NEXT: pxor %xmm10, %xmm10 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3] |
| ; SSE-NEXT: pxor %xmm11, %xmm11 |
| ; SSE-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7] |
| ; SSE-NEXT: pxor %xmm12, %xmm12 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm5[0],xmm12[1],xmm5[1],xmm12[2],xmm5[2],xmm12[3],xmm5[3] |
| ; SSE-NEXT: pxor %xmm13, %xmm13 |
| ; SSE-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm5[4],xmm13[5],xmm5[5],xmm13[6],xmm5[6],xmm13[7],xmm5[7] |
| ; SSE-NEXT: pxor %xmm14, %xmm14 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3] |
| ; SSE-NEXT: pxor %xmm15, %xmm15 |
| ; SSE-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm6[4],xmm15[5],xmm6[5],xmm15[6],xmm6[6],xmm15[7],xmm6[7] |
| ; SSE-NEXT: pxor %xmm8, %xmm8 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3] |
| ; SSE-NEXT: pxor %xmm4, %xmm4 |
| ; SSE-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7] |
| ; SSE-NEXT: pxor %xmm5, %xmm5 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3] |
| ; SSE-NEXT: mulps %xmm10, %xmm5 |
| ; SSE-NEXT: pxor %xmm6, %xmm6 |
| ; SSE-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm9[4],xmm6[5],xmm9[5],xmm6[6],xmm9[6],xmm6[7],xmm9[7] |
| ; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm7 |
| ; SSE-NEXT: mulps %xmm11, %xmm6 |
| ; SSE-NEXT: pxor %xmm9, %xmm9 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3] |
| ; SSE-NEXT: mulps %xmm12, %xmm9 |
| ; SSE-NEXT: pxor %xmm10, %xmm10 |
| ; SSE-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm7[4],xmm10[5],xmm7[5],xmm10[6],xmm7[6],xmm10[7],xmm7[7] |
| ; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm7 |
| ; SSE-NEXT: mulps %xmm13, %xmm10 |
| ; SSE-NEXT: pxor %xmm11, %xmm11 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm7[0],xmm11[1],xmm7[1],xmm11[2],xmm7[2],xmm11[3],xmm7[3] |
| ; SSE-NEXT: mulps %xmm14, %xmm11 |
| ; SSE-NEXT: pxor %xmm12, %xmm12 |
| ; SSE-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7] |
| ; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm7 |
| ; SSE-NEXT: mulps %xmm15, %xmm12 |
| ; SSE-NEXT: pxor %xmm13, %xmm13 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm13 = xmm13[0],xmm7[0],xmm13[1],xmm7[1],xmm13[2],xmm7[2],xmm13[3],xmm7[3] |
| ; SSE-NEXT: mulps %xmm8, %xmm13 |
| ; SSE-NEXT: pxor %xmm8, %xmm8 |
| ; SSE-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7] |
| ; SSE-NEXT: mulps %xmm4, %xmm8 |
| ; SSE-NEXT: addps %xmm5, %xmm0 |
| ; SSE-NEXT: addps %xmm6, %xmm0 |
| ; SSE-NEXT: addps %xmm9, %xmm1 |
| ; SSE-NEXT: addps %xmm10, %xmm1 |
| ; SSE-NEXT: addps %xmm11, %xmm2 |
| ; SSE-NEXT: addps %xmm12, %xmm2 |
| ; SSE-NEXT: addps %xmm13, %xmm3 |
| ; SSE-NEXT: addps %xmm8, %xmm3 |
| ; SSE-NEXT: retq |
| ; |
| ; AVX2-LABEL: partial_reduce_fmla_bf16_v16f32: |
| ; AVX2: # %bb.0: |
| ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero |
| ; AVX2-NEXT: vpslld $16, %ymm6, %ymm6 |
| ; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm2 |
| ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero |
| ; AVX2-NEXT: vpslld $16, %ymm2, %ymm2 |
| ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm7 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero |
| ; AVX2-NEXT: vpslld $16, %ymm7, %ymm7 |
| ; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm3 |
| ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero |
| ; AVX2-NEXT: vpslld $16, %ymm3, %ymm3 |
| ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm8 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero |
| ; AVX2-NEXT: vpslld $16, %ymm8, %ymm8 |
| ; AVX2-NEXT: vmulps %ymm6, %ymm8, %ymm6 |
| ; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm4 |
| ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero |
| ; AVX2-NEXT: vpslld $16, %ymm4, %ymm4 |
| ; AVX2-NEXT: vmulps %ymm4, %ymm2, %ymm2 |
| ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero |
| ; AVX2-NEXT: vpslld $16, %ymm4, %ymm4 |
| ; AVX2-NEXT: vmulps %ymm4, %ymm7, %ymm4 |
| ; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm5 |
| ; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero |
| ; AVX2-NEXT: vpslld $16, %ymm5, %ymm5 |
| ; AVX2-NEXT: vmulps %ymm5, %ymm3, %ymm3 |
| ; AVX2-NEXT: vaddps %ymm6, %ymm0, %ymm0 |
| ; AVX2-NEXT: vaddps %ymm2, %ymm0, %ymm0 |
| ; AVX2-NEXT: vaddps %ymm4, %ymm1, %ymm1 |
| ; AVX2-NEXT: vaddps %ymm3, %ymm1, %ymm1 |
| ; AVX2-NEXT: retq |
| ; |
| ; AVX512-LABEL: partial_reduce_fmla_bf16_v16f32: |
| ; AVX512: # %bb.0: |
| ; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero |
| ; AVX512-NEXT: vpslld $16, %ymm6, %ymm6 |
| ; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm2 |
| ; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero |
| ; AVX512-NEXT: vpslld $16, %ymm2, %ymm2 |
| ; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm7 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero |
| ; AVX512-NEXT: vpslld $16, %ymm7, %ymm7 |
| ; AVX512-NEXT: vextracti128 $1, %ymm3, %xmm3 |
| ; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero |
| ; AVX512-NEXT: vpslld $16, %ymm3, %ymm3 |
| ; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm8 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero |
| ; AVX512-NEXT: vpslld $16, %ymm8, %ymm8 |
| ; AVX512-NEXT: vmulps %ymm6, %ymm8, %ymm6 |
| ; AVX512-NEXT: vextracti128 $1, %ymm4, %xmm4 |
| ; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero |
| ; AVX512-NEXT: vpslld $16, %ymm4, %ymm4 |
| ; AVX512-NEXT: vmulps %ymm4, %ymm2, %ymm2 |
| ; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero |
| ; AVX512-NEXT: vpslld $16, %ymm4, %ymm4 |
| ; AVX512-NEXT: vmulps %ymm4, %ymm7, %ymm4 |
| ; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm5 |
| ; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero |
| ; AVX512-NEXT: vpslld $16, %ymm5, %ymm5 |
| ; AVX512-NEXT: vmulps %ymm5, %ymm3, %ymm3 |
| ; AVX512-NEXT: vaddps %ymm6, %ymm0, %ymm0 |
| ; AVX512-NEXT: vaddps %ymm2, %ymm0, %ymm0 |
| ; AVX512-NEXT: vaddps %ymm4, %ymm1, %ymm1 |
| ; AVX512-NEXT: vaddps %ymm3, %ymm1, %ymm1 |
| ; AVX512-NEXT: retq |
| ; |
| ; AVX512BF16-LABEL: partial_reduce_fmla_bf16_v16f32: |
| ; AVX512BF16: # %bb.0: |
| ; AVX512BF16-NEXT: vdpbf16ps %zmm2, %zmm1, %zmm0 |
| ; AVX512BF16-NEXT: retq |
| ; |
| ; AVX10-512-LABEL: partial_reduce_fmla_bf16_v16f32: |
| ; AVX10-512: # %bb.0: |
| ; AVX10-512-NEXT: vdpbf16ps %zmm2, %zmm1, %zmm0 |
| ; AVX10-512-NEXT: retq |
| ; |
| ; AVX10-256-LABEL: partial_reduce_fmla_bf16_v16f32: |
| ; AVX10-256: # %bb.0: |
| ; AVX10-256-NEXT: vdpbf16ps %ymm4, %ymm2, %ymm0 |
| ; AVX10-256-NEXT: vdpbf16ps %ymm5, %ymm3, %ymm1 |
| ; AVX10-256-NEXT: retq |
| %a.ext = fpext <32 x bfloat> %a to <32 x float> |
| %b.ext = fpext <32 x bfloat> %b to <32 x float> |
| %mul = fmul <32 x float> %a.ext, %b.ext |
| %res = call <16 x float> @llvm.vector.partial.reduce.fadd.v16f32.v32f32(<16 x float> %acc, <32 x float> %mul) |
| ret <16 x float> %res |
| } |
| |
| ; Accumulators narrower than 128 bits are widened by expanding the partial |
| ; reduction. |
| |
| define <2 x float> @partial_reduce_fmla_bf16_v2f32(<2 x float> %acc, <4 x bfloat> %a, <4 x bfloat> %b) #0 { |
| ; SSE-LABEL: partial_reduce_fmla_bf16_v2f32: |
| ; SSE: # %bb.0: |
| ; SSE-NEXT: pxor %xmm3, %xmm3 |
| ; SSE-NEXT: pxor %xmm4, %xmm4 |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3] |
| ; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] |
| ; SSE-NEXT: mulps %xmm4, %xmm3 |
| ; SSE-NEXT: addps %xmm3, %xmm0 |
| ; SSE-NEXT: movhlps {{.*#+}} xmm3 = xmm3[1,1] |
| ; SSE-NEXT: addps %xmm3, %xmm0 |
| ; SSE-NEXT: retq |
| ; |
| ; AVX2-LABEL: partial_reduce_fmla_bf16_v2f32: |
| ; AVX2: # %bb.0: |
| ; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3 |
| ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3] |
| ; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] |
| ; AVX2-NEXT: vmulps %xmm2, %xmm1, %xmm1 |
| ; AVX2-NEXT: vaddps %xmm1, %xmm0, %xmm0 |
| ; AVX2-NEXT: vshufpd {{.*#+}} xmm1 = xmm1[1,0] |
| ; AVX2-NEXT: vaddps %xmm0, %xmm1, %xmm0 |
| ; AVX2-NEXT: retq |
| ; |
| ; AVX512-LABEL: partial_reduce_fmla_bf16_v2f32: |
| ; AVX512: # %bb.0: |
| ; AVX512-NEXT: vpxor %xmm3, %xmm3, %xmm3 |
| ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3] |
| ; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] |
| ; AVX512-NEXT: vmulps %xmm2, %xmm1, %xmm1 |
| ; AVX512-NEXT: vaddps %xmm1, %xmm0, %xmm0 |
| ; AVX512-NEXT: vshufpd {{.*#+}} xmm1 = xmm1[1,0] |
| ; AVX512-NEXT: vaddps %xmm0, %xmm1, %xmm0 |
| ; AVX512-NEXT: retq |
| ; |
| ; AVX512BF16-LABEL: partial_reduce_fmla_bf16_v2f32: |
| ; AVX512BF16: # %bb.0: |
| ; AVX512BF16-NEXT: vpxor %xmm3, %xmm3, %xmm3 |
| ; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] |
| ; AVX512BF16-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3] |
| ; AVX512BF16-NEXT: vmulps %xmm2, %xmm1, %xmm1 |
| ; AVX512BF16-NEXT: vaddps %xmm1, %xmm0, %xmm0 |
| ; AVX512BF16-NEXT: vshufpd {{.*#+}} xmm1 = xmm1[1,0] |
| ; AVX512BF16-NEXT: vaddps %xmm0, %xmm1, %xmm0 |
| ; AVX512BF16-NEXT: retq |
| ; |
| ; AVX10-LABEL: partial_reduce_fmla_bf16_v2f32: |
| ; AVX10: # %bb.0: |
| ; AVX10-NEXT: vpxor %xmm3, %xmm3, %xmm3 |
| ; AVX10-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] |
| ; AVX10-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3] |
| ; AVX10-NEXT: vmulps %xmm2, %xmm1, %xmm1 |
| ; AVX10-NEXT: vaddps %xmm1, %xmm0, %xmm0 |
| ; AVX10-NEXT: vshufpd {{.*#+}} xmm1 = xmm1[1,0] |
| ; AVX10-NEXT: vaddps %xmm0, %xmm1, %xmm0 |
| ; AVX10-NEXT: retq |
| %a.ext = fpext <4 x bfloat> %a to <4 x float> |
| %b.ext = fpext <4 x bfloat> %b to <4 x float> |
| %mul = fmul <4 x float> %a.ext, %b.ext |
| %res = call <2 x float> @llvm.vector.partial.reduce.fadd.v2f32.v4f32(<2 x float> %acc, <4 x float> %mul) |
| ret <2 x float> %res |
| } |
| |
| attributes #0 = { "min-legal-vector-width"="256" } |
