| ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 | |
| ; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s --check-prefixes=SSE | |
| ; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX | |
| ;; bf16 splats are lowered as vXi16 shuffles, but combineTargetShuffle turns | |
| ;; VBROADCAST(bitcast(x)) into bitcast(VBROADCAST(x)) when x is a legal bf16 | |
| ;; vector, e.g. a value that is live across basic blocks or a volatile load. | |
| ;; FIXME: AVX2 targets without AVX512BW+VLX crash on these, so they are not | |
| ;; tested yet. | |
| define void @splat_v8bf16_cross_block(<8 x bfloat> %v, i1 %c, ptr %p) nounwind { | |
| ; SSE-LABEL: splat_v8bf16_cross_block: | |
| ; SSE: # %bb.0: # %entry | |
| ; SSE-NEXT: testb $1, %dil | |
| ; SSE-NEXT: je .LBB0_2 | |
| ; SSE-NEXT: # %bb.1: # %splat | |
| ; SSE-NEXT: pslld $16, %xmm0 | |
| ; SSE-NEXT: movd %xmm0, %eax | |
| ; SSE-NEXT: shrl $16, %eax | |
| ; SSE-NEXT: movw %ax, 14(%rsi) | |
| ; SSE-NEXT: movw %ax, 12(%rsi) | |
| ; SSE-NEXT: movw %ax, 10(%rsi) | |
| ; SSE-NEXT: movw %ax, 8(%rsi) | |
| ; SSE-NEXT: movw %ax, 6(%rsi) | |
| ; SSE-NEXT: movw %ax, 4(%rsi) | |
| ; SSE-NEXT: movw %ax, 2(%rsi) | |
| ; SSE-NEXT: movw %ax, (%rsi) | |
| ; SSE-NEXT: .LBB0_2: # %exit | |
| ; SSE-NEXT: retq | |
| ; | |
| ; AVX-LABEL: splat_v8bf16_cross_block: | |
| ; AVX: # %bb.0: # %entry | |
| ; AVX-NEXT: testb $1, %dil | |
| ; AVX-NEXT: je .LBB0_2 | |
| ; AVX-NEXT: # %bb.1: # %splat | |
| ; AVX-NEXT: vpbroadcastw %xmm0, %xmm0 | |
| ; AVX-NEXT: vmovdqa %xmm0, (%rsi) | |
| ; AVX-NEXT: .LBB0_2: # %exit | |
| ; AVX-NEXT: retq | |
| entry: | |
| br i1 %c, label %splat, label %exit | |
| splat: | |
| %s = shufflevector <8 x bfloat> %v, <8 x bfloat> poison, <8 x i32> zeroinitializer | |
| store <8 x bfloat> %s, ptr %p | |
| br label %exit | |
| exit: | |
| ret void | |
| } | |
| define void @splat_v16bf16_from_v8bf16_cross_block(<8 x bfloat> %v, i1 %c, ptr %p) nounwind { | |
| ; SSE-LABEL: splat_v16bf16_from_v8bf16_cross_block: | |
| ; SSE: # %bb.0: # %entry | |
| ; SSE-NEXT: testb $1, %dil | |
| ; SSE-NEXT: je .LBB1_2 | |
| ; SSE-NEXT: # %bb.1: # %splat | |
| ; SSE-NEXT: pslld $16, %xmm0 | |
| ; SSE-NEXT: movd %xmm0, %eax | |
| ; SSE-NEXT: shrl $16, %eax | |
| ; SSE-NEXT: movw %ax, 30(%rsi) | |
| ; SSE-NEXT: movw %ax, 28(%rsi) | |
| ; SSE-NEXT: movw %ax, 26(%rsi) | |
| ; SSE-NEXT: movw %ax, 24(%rsi) | |
| ; SSE-NEXT: movw %ax, 22(%rsi) | |
| ; SSE-NEXT: movw %ax, 20(%rsi) | |
| ; SSE-NEXT: movw %ax, 18(%rsi) | |
| ; SSE-NEXT: movw %ax, 16(%rsi) | |
| ; SSE-NEXT: movw %ax, 14(%rsi) | |
| ; SSE-NEXT: movw %ax, 12(%rsi) | |
| ; SSE-NEXT: movw %ax, 10(%rsi) | |
| ; SSE-NEXT: movw %ax, 8(%rsi) | |
| ; SSE-NEXT: movw %ax, 6(%rsi) | |
| ; SSE-NEXT: movw %ax, 4(%rsi) | |
| ; SSE-NEXT: movw %ax, 2(%rsi) | |
| ; SSE-NEXT: movw %ax, (%rsi) | |
| ; SSE-NEXT: .LBB1_2: # %exit | |
| ; SSE-NEXT: retq | |
| ; | |
| ; AVX-LABEL: splat_v16bf16_from_v8bf16_cross_block: | |
| ; AVX: # %bb.0: # %entry | |
| ; AVX-NEXT: testb $1, %dil | |
| ; AVX-NEXT: je .LBB1_2 | |
| ; AVX-NEXT: # %bb.1: # %splat | |
| ; AVX-NEXT: vpbroadcastw %xmm0, %ymm0 | |
| ; AVX-NEXT: vmovdqa %ymm0, (%rsi) | |
| ; AVX-NEXT: .LBB1_2: # %exit | |
| ; AVX-NEXT: vzeroupper | |
| ; AVX-NEXT: retq | |
| entry: | |
| br i1 %c, label %splat, label %exit | |
| splat: | |
| %s = shufflevector <8 x bfloat> %v, <8 x bfloat> poison, <16 x i32> zeroinitializer | |
| store <16 x bfloat> %s, ptr %p | |
| br label %exit | |
| exit: | |
| ret void | |
| } | |
| define void @splat_v16bf16_cross_block(<16 x bfloat> %v, i1 %c, ptr %p) nounwind { | |
| ; SSE-LABEL: splat_v16bf16_cross_block: | |
| ; SSE: # %bb.0: # %entry | |
| ; SSE-NEXT: testb $1, %dil | |
| ; SSE-NEXT: je .LBB2_2 | |
| ; SSE-NEXT: # %bb.1: # %splat | |
| ; SSE-NEXT: pslld $16, %xmm0 | |
| ; SSE-NEXT: movd %xmm0, %eax | |
| ; SSE-NEXT: shrl $16, %eax | |
| ; SSE-NEXT: movw %ax, 30(%rsi) | |
| ; SSE-NEXT: movw %ax, 28(%rsi) | |
| ; SSE-NEXT: movw %ax, 26(%rsi) | |
| ; SSE-NEXT: movw %ax, 24(%rsi) | |
| ; SSE-NEXT: movw %ax, 22(%rsi) | |
| ; SSE-NEXT: movw %ax, 20(%rsi) | |
| ; SSE-NEXT: movw %ax, 18(%rsi) | |
| ; SSE-NEXT: movw %ax, 16(%rsi) | |
| ; SSE-NEXT: movw %ax, 14(%rsi) | |
| ; SSE-NEXT: movw %ax, 12(%rsi) | |
| ; SSE-NEXT: movw %ax, 10(%rsi) | |
| ; SSE-NEXT: movw %ax, 8(%rsi) | |
| ; SSE-NEXT: movw %ax, 6(%rsi) | |
| ; SSE-NEXT: movw %ax, 4(%rsi) | |
| ; SSE-NEXT: movw %ax, 2(%rsi) | |
| ; SSE-NEXT: movw %ax, (%rsi) | |
| ; SSE-NEXT: .LBB2_2: # %exit | |
| ; SSE-NEXT: retq | |
| ; | |
| ; AVX-LABEL: splat_v16bf16_cross_block: | |
| ; AVX: # %bb.0: # %entry | |
| ; AVX-NEXT: testb $1, %dil | |
| ; AVX-NEXT: je .LBB2_2 | |
| ; AVX-NEXT: # %bb.1: # %splat | |
| ; AVX-NEXT: vpbroadcastw %xmm0, %ymm0 | |
| ; AVX-NEXT: vmovdqa %ymm0, (%rsi) | |
| ; AVX-NEXT: .LBB2_2: # %exit | |
| ; AVX-NEXT: vzeroupper | |
| ; AVX-NEXT: retq | |
| entry: | |
| br i1 %c, label %splat, label %exit | |
| splat: | |
| %s = shufflevector <16 x bfloat> %v, <16 x bfloat> poison, <16 x i32> zeroinitializer | |
| store <16 x bfloat> %s, ptr %p | |
| br label %exit | |
| exit: | |
| ret void | |
| } | |
| ;; A volatile load cannot be narrowed into a 16-bit broadcast load, so this | |
| ;; needs a register broadcast even within a single block. | |
| define void @splat_v8bf16_volatile_load(ptr %src, ptr %dst) nounwind { | |
| ; SSE-LABEL: splat_v8bf16_volatile_load: | |
| ; SSE: # %bb.0: | |
| ; SSE-NEXT: movzwl 14(%rdi), %eax | |
| ; SSE-NEXT: movzwl 12(%rdi), %eax | |
| ; SSE-NEXT: movzwl 10(%rdi), %eax | |
| ; SSE-NEXT: movzwl 8(%rdi), %eax | |
| ; SSE-NEXT: movzwl 6(%rdi), %eax | |
| ; SSE-NEXT: movzwl 4(%rdi), %eax | |
| ; SSE-NEXT: movzwl (%rdi), %eax | |
| ; SSE-NEXT: movzwl 2(%rdi), %ecx | |
| ; SSE-NEXT: movw %ax, 14(%rsi) | |
| ; SSE-NEXT: movw %ax, 12(%rsi) | |
| ; SSE-NEXT: movw %ax, 10(%rsi) | |
| ; SSE-NEXT: movw %ax, 8(%rsi) | |
| ; SSE-NEXT: movw %ax, 6(%rsi) | |
| ; SSE-NEXT: movw %ax, 4(%rsi) | |
| ; SSE-NEXT: movw %ax, 2(%rsi) | |
| ; SSE-NEXT: movw %ax, (%rsi) | |
| ; SSE-NEXT: retq | |
| ; | |
| ; AVX-LABEL: splat_v8bf16_volatile_load: | |
| ; AVX: # %bb.0: | |
| ; AVX-NEXT: vmovdqa (%rdi), %xmm0 | |
| ; AVX-NEXT: vpbroadcastw %xmm0, %xmm0 | |
| ; AVX-NEXT: vmovdqa %xmm0, (%rsi) | |
| ; AVX-NEXT: retq | |
| %v = load volatile <8 x bfloat>, ptr %src | |
| %s = shufflevector <8 x bfloat> %v, <8 x bfloat> poison, <8 x i32> zeroinitializer | |
| store <8 x bfloat> %s, ptr %dst | |
| ret void | |
| } | |
| ;; A load whose other uses are simplified away after legalization is folded | |
| ;; into a bf16 VBROADCAST_LOAD. | |
| define void @splat_v8bf16_broadcast_load(ptr %p, <8 x bfloat> %other, ptr %dst1, ptr %dst2) nounwind { | |
| ; SSE-LABEL: splat_v8bf16_broadcast_load: | |
| ; SSE: # %bb.0: | |
| ; SSE-NEXT: movq %xmm0, %rax | |
| ; SSE-NEXT: movq %rax, %rcx | |
| ; SSE-NEXT: movq %rax, %r8 | |
| ; SSE-NEXT: movzwl (%rdi), %edi | |
| ; SSE-NEXT: movw %di, 8(%rsi) | |
| ; SSE-NEXT: movw %di, (%rsi) | |
| ; SSE-NEXT: movw %di, 14(%rsi) | |
| ; SSE-NEXT: movw %di, 12(%rsi) | |
| ; SSE-NEXT: movw %di, 6(%rsi) | |
| ; SSE-NEXT: movw %di, 4(%rsi) | |
| ; SSE-NEXT: movw %di, 10(%rsi) | |
| ; SSE-NEXT: movw %di, 2(%rsi) | |
| ; SSE-NEXT: movw %ax, 8(%rdx) | |
| ; SSE-NEXT: movw %ax, (%rdx) | |
| ; SSE-NEXT: # kill: def $eax killed $eax killed $rax | |
| ; SSE-NEXT: shrl $16, %eax | |
| ; SSE-NEXT: shrq $32, %rcx | |
| ; SSE-NEXT: shrq $48, %r8 | |
| ; SSE-NEXT: movw %r8w, 14(%rdx) | |
| ; SSE-NEXT: movw %cx, 12(%rdx) | |
| ; SSE-NEXT: movw %r8w, 6(%rdx) | |
| ; SSE-NEXT: movw %cx, 4(%rdx) | |
| ; SSE-NEXT: movw %ax, 10(%rdx) | |
| ; SSE-NEXT: movw %ax, 2(%rdx) | |
| ; SSE-NEXT: retq | |
| ; | |
| ; AVX-LABEL: splat_v8bf16_broadcast_load: | |
| ; AVX: # %bb.0: | |
| ; AVX-NEXT: vpbroadcastw (%rdi), %xmm1 | |
| ; AVX-NEXT: vmovdqa %xmm1, (%rsi) | |
| ; AVX-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0] | |
| ; AVX-NEXT: vmovaps %xmm0, (%rdx) | |
| ; AVX-NEXT: retq | |
| %ld = load <8 x bfloat>, ptr %p | |
| %shuf = shufflevector <8 x bfloat> %ld, <8 x bfloat> %other, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11> | |
| %splat0 = shufflevector <8 x bfloat> %shuf, <8 x bfloat> poison, <8 x i32> zeroinitializer | |
| store <8 x bfloat> %splat0, ptr %dst1 | |
| %other_lo = shufflevector <8 x bfloat> %shuf, <8 x bfloat> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7> | |
| store <8 x bfloat> %other_lo, ptr %dst2 | |
| ret void | |
| } | |
| define void @splat_v16bf16_broadcast_load(ptr %p, <8 x bfloat> %other, ptr %dst1, ptr %dst2) nounwind { | |
| ; SSE-LABEL: splat_v16bf16_broadcast_load: | |
| ; SSE: # %bb.0: | |
| ; SSE-NEXT: movq %xmm0, %rax | |
| ; SSE-NEXT: movq %rax, %rcx | |
| ; SSE-NEXT: movq %rax, %r8 | |
| ; SSE-NEXT: movzwl (%rdi), %edi | |
| ; SSE-NEXT: movw %di, 28(%rsi) | |
| ; SSE-NEXT: movw %di, 26(%rsi) | |
| ; SSE-NEXT: movw %di, 24(%rsi) | |
| ; SSE-NEXT: movw %di, 22(%rsi) | |
| ; SSE-NEXT: movw %di, 20(%rsi) | |
| ; SSE-NEXT: movw %di, 18(%rsi) | |
| ; SSE-NEXT: movw %di, 16(%rsi) | |
| ; SSE-NEXT: movw %di, 30(%rsi) | |
| ; SSE-NEXT: movw %ax, 8(%rdx) | |
| ; SSE-NEXT: movw %ax, (%rdx) | |
| ; SSE-NEXT: # kill: def $eax killed $eax killed $rax | |
| ; SSE-NEXT: shrl $16, %eax | |
| ; SSE-NEXT: shrq $32, %rcx | |
| ; SSE-NEXT: shrq $48, %r8 | |
| ; SSE-NEXT: movw %r8w, 14(%rdx) | |
| ; SSE-NEXT: movw %cx, 12(%rdx) | |
| ; SSE-NEXT: movw %r8w, 6(%rdx) | |
| ; SSE-NEXT: movw %cx, 4(%rdx) | |
| ; SSE-NEXT: movw %ax, 10(%rdx) | |
| ; SSE-NEXT: movw %ax, 2(%rdx) | |
| ; SSE-NEXT: retq | |
| ; | |
| ; AVX-LABEL: splat_v16bf16_broadcast_load: | |
| ; AVX: # %bb.0: | |
| ; AVX-NEXT: vpbroadcastw (%rdi), %ymm1 | |
| ; AVX-NEXT: vmovdqa %ymm1, (%rsi) | |
| ; AVX-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0] | |
| ; AVX-NEXT: vmovaps %xmm0, (%rdx) | |
| ; AVX-NEXT: vzeroupper | |
| ; AVX-NEXT: retq | |
| %ld = load <8 x bfloat>, ptr %p | |
| %shuf = shufflevector <8 x bfloat> %ld, <8 x bfloat> %other, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11> | |
| %splat0 = shufflevector <8 x bfloat> %shuf, <8 x bfloat> poison, <8 x i32> zeroinitializer | |
| %wide = shufflevector <8 x bfloat> poison, <8 x bfloat> %splat0, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> | |
| store <16 x bfloat> %wide, ptr %dst1 | |
| %other_lo = shufflevector <8 x bfloat> %shuf, <8 x bfloat> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7> | |
| store <8 x bfloat> %other_lo, ptr %dst2 | |
| ret void | |
| } | |
| ;; combineTargetShuffle must not look through a bitcast from v8bf16 to form a | |
| ;; broadcast of an illegal bf16 vector type (v16bf16 without AVX-NE-CONVERT or | |
| ;; AVX512BW). | |
| define void @splat_v16i16_from_bitcast_v8bf16(<8 x bfloat> %v, i1 %c, ptr %dst) nounwind { | |
| ; SSE-LABEL: splat_v16i16_from_bitcast_v8bf16: | |
| ; SSE: # %bb.0: # %entry | |
| ; SSE-NEXT: testb $1, %dil | |
| ; SSE-NEXT: je .LBB6_2 | |
| ; SSE-NEXT: # %bb.1: # %splat | |
| ; SSE-NEXT: pslld $16, %xmm0 | |
| ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] | |
| ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] | |
| ; SSE-NEXT: movdqa %xmm0, 16(%rsi) | |
| ; SSE-NEXT: movdqa %xmm0, (%rsi) | |
| ; SSE-NEXT: .LBB6_2: # %exit | |
| ; SSE-NEXT: retq | |
| ; | |
| ; AVX-LABEL: splat_v16i16_from_bitcast_v8bf16: | |
| ; AVX: # %bb.0: # %entry | |
| ; AVX-NEXT: testb $1, %dil | |
| ; AVX-NEXT: je .LBB6_2 | |
| ; AVX-NEXT: # %bb.1: # %splat | |
| ; AVX-NEXT: vpbroadcastw %xmm0, %ymm0 | |
| ; AVX-NEXT: vmovdqa %ymm0, (%rsi) | |
| ; AVX-NEXT: .LBB6_2: # %exit | |
| ; AVX-NEXT: vzeroupper | |
| ; AVX-NEXT: retq | |
| entry: | |
| br i1 %c, label %splat, label %exit | |
| splat: | |
| %bc = bitcast <8 x bfloat> %v to <8 x i16> | |
| %s = shufflevector <8 x i16> %bc, <8 x i16> poison, <16 x i32> zeroinitializer | |
| store <16 x i16> %s, ptr %dst | |
| br label %exit | |
| exit: | |
| ret void | |
| } | |
| define void @rebroadcast_v8i32_from_v8bf16(ptr %src, ptr %dst1, ptr %dst2) nounwind { | |
| ; SSE-LABEL: rebroadcast_v8i32_from_v8bf16: | |
| ; SSE: # %bb.0: | |
| ; SSE-NEXT: movzwl 14(%rdi), %eax | |
| ; SSE-NEXT: movzwl 12(%rdi), %eax | |
| ; SSE-NEXT: movzwl 10(%rdi), %eax | |
| ; SSE-NEXT: movzwl 8(%rdi), %eax | |
| ; SSE-NEXT: movzwl 6(%rdi), %eax | |
| ; SSE-NEXT: movzwl 2(%rdi), %eax | |
| ; SSE-NEXT: movzwl 4(%rdi), %eax | |
| ; SSE-NEXT: movzwl (%rdi), %eax | |
| ; SSE-NEXT: movl %eax, %ecx | |
| ; SSE-NEXT: shll $16, %ecx | |
| ; SSE-NEXT: orl %eax, %ecx | |
| ; SSE-NEXT: movw %ax, 14(%rsi) | |
| ; SSE-NEXT: movw %ax, 12(%rsi) | |
| ; SSE-NEXT: movw %ax, 10(%rsi) | |
| ; SSE-NEXT: movw %ax, 8(%rsi) | |
| ; SSE-NEXT: movw %ax, 6(%rsi) | |
| ; SSE-NEXT: movw %ax, 4(%rsi) | |
| ; SSE-NEXT: movw %ax, 2(%rsi) | |
| ; SSE-NEXT: movw %ax, (%rsi) | |
| ; SSE-NEXT: movd %ecx, %xmm0 | |
| ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] | |
| ; SSE-NEXT: movdqa %xmm0, 16(%rdx) | |
| ; SSE-NEXT: movdqa %xmm0, (%rdx) | |
| ; SSE-NEXT: retq | |
| ; | |
| ; AVX-LABEL: rebroadcast_v8i32_from_v8bf16: | |
| ; AVX: # %bb.0: | |
| ; AVX-NEXT: vmovdqa (%rdi), %xmm0 | |
| ; AVX-NEXT: vpbroadcastw %xmm0, %ymm0 | |
| ; AVX-NEXT: vmovdqa %xmm0, (%rsi) | |
| ; AVX-NEXT: vmovdqa %ymm0, (%rdx) | |
| ; AVX-NEXT: vzeroupper | |
| ; AVX-NEXT: retq | |
| %v = load volatile <8 x bfloat>, ptr %src | |
| %s8 = shufflevector <8 x bfloat> %v, <8 x bfloat> poison, <8 x i32> zeroinitializer | |
| store <8 x bfloat> %s8, ptr %dst1 | |
| %bc = bitcast <8 x bfloat> %s8 to <4 x i32> | |
| %s256 = shufflevector <4 x i32> %bc, <4 x i32> poison, <8 x i32> zeroinitializer | |
| store <8 x i32> %s256, ptr %dst2 | |
| ret void | |
| } |