blob: 9aa00cabc85c5d00e743dd5df8dd50021f6b7fb5 [file] [edit]
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s --check-prefixes=SSE
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX
;; bf16 splats are lowered as vXi16 shuffles, but combineTargetShuffle turns
;; VBROADCAST(bitcast(x)) into bitcast(VBROADCAST(x)) when x is a legal bf16
;; vector, e.g. a value that is live across basic blocks or a volatile load.
;; FIXME: AVX2 targets without AVX512BW+VLX crash on these, so they are not
;; tested yet.
define void @splat_v8bf16_cross_block(<8 x bfloat> %v, i1 %c, ptr %p) nounwind {
; SSE-LABEL: splat_v8bf16_cross_block:
; SSE: # %bb.0: # %entry
; SSE-NEXT: testb $1, %dil
; SSE-NEXT: je .LBB0_2
; SSE-NEXT: # %bb.1: # %splat
; SSE-NEXT: pslld $16, %xmm0
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: shrl $16, %eax
; SSE-NEXT: movw %ax, 14(%rsi)
; SSE-NEXT: movw %ax, 12(%rsi)
; SSE-NEXT: movw %ax, 10(%rsi)
; SSE-NEXT: movw %ax, 8(%rsi)
; SSE-NEXT: movw %ax, 6(%rsi)
; SSE-NEXT: movw %ax, 4(%rsi)
; SSE-NEXT: movw %ax, 2(%rsi)
; SSE-NEXT: movw %ax, (%rsi)
; SSE-NEXT: .LBB0_2: # %exit
; SSE-NEXT: retq
;
; AVX-LABEL: splat_v8bf16_cross_block:
; AVX: # %bb.0: # %entry
; AVX-NEXT: testb $1, %dil
; AVX-NEXT: je .LBB0_2
; AVX-NEXT: # %bb.1: # %splat
; AVX-NEXT: vpbroadcastw %xmm0, %xmm0
; AVX-NEXT: vmovdqa %xmm0, (%rsi)
; AVX-NEXT: .LBB0_2: # %exit
; AVX-NEXT: retq
entry:
br i1 %c, label %splat, label %exit
splat:
%s = shufflevector <8 x bfloat> %v, <8 x bfloat> poison, <8 x i32> zeroinitializer
store <8 x bfloat> %s, ptr %p
br label %exit
exit:
ret void
}
define void @splat_v16bf16_from_v8bf16_cross_block(<8 x bfloat> %v, i1 %c, ptr %p) nounwind {
; SSE-LABEL: splat_v16bf16_from_v8bf16_cross_block:
; SSE: # %bb.0: # %entry
; SSE-NEXT: testb $1, %dil
; SSE-NEXT: je .LBB1_2
; SSE-NEXT: # %bb.1: # %splat
; SSE-NEXT: pslld $16, %xmm0
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: shrl $16, %eax
; SSE-NEXT: movw %ax, 30(%rsi)
; SSE-NEXT: movw %ax, 28(%rsi)
; SSE-NEXT: movw %ax, 26(%rsi)
; SSE-NEXT: movw %ax, 24(%rsi)
; SSE-NEXT: movw %ax, 22(%rsi)
; SSE-NEXT: movw %ax, 20(%rsi)
; SSE-NEXT: movw %ax, 18(%rsi)
; SSE-NEXT: movw %ax, 16(%rsi)
; SSE-NEXT: movw %ax, 14(%rsi)
; SSE-NEXT: movw %ax, 12(%rsi)
; SSE-NEXT: movw %ax, 10(%rsi)
; SSE-NEXT: movw %ax, 8(%rsi)
; SSE-NEXT: movw %ax, 6(%rsi)
; SSE-NEXT: movw %ax, 4(%rsi)
; SSE-NEXT: movw %ax, 2(%rsi)
; SSE-NEXT: movw %ax, (%rsi)
; SSE-NEXT: .LBB1_2: # %exit
; SSE-NEXT: retq
;
; AVX-LABEL: splat_v16bf16_from_v8bf16_cross_block:
; AVX: # %bb.0: # %entry
; AVX-NEXT: testb $1, %dil
; AVX-NEXT: je .LBB1_2
; AVX-NEXT: # %bb.1: # %splat
; AVX-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX-NEXT: vmovdqa %ymm0, (%rsi)
; AVX-NEXT: .LBB1_2: # %exit
; AVX-NEXT: vzeroupper
; AVX-NEXT: retq
entry:
br i1 %c, label %splat, label %exit
splat:
%s = shufflevector <8 x bfloat> %v, <8 x bfloat> poison, <16 x i32> zeroinitializer
store <16 x bfloat> %s, ptr %p
br label %exit
exit:
ret void
}
define void @splat_v16bf16_cross_block(<16 x bfloat> %v, i1 %c, ptr %p) nounwind {
; SSE-LABEL: splat_v16bf16_cross_block:
; SSE: # %bb.0: # %entry
; SSE-NEXT: testb $1, %dil
; SSE-NEXT: je .LBB2_2
; SSE-NEXT: # %bb.1: # %splat
; SSE-NEXT: pslld $16, %xmm0
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: shrl $16, %eax
; SSE-NEXT: movw %ax, 30(%rsi)
; SSE-NEXT: movw %ax, 28(%rsi)
; SSE-NEXT: movw %ax, 26(%rsi)
; SSE-NEXT: movw %ax, 24(%rsi)
; SSE-NEXT: movw %ax, 22(%rsi)
; SSE-NEXT: movw %ax, 20(%rsi)
; SSE-NEXT: movw %ax, 18(%rsi)
; SSE-NEXT: movw %ax, 16(%rsi)
; SSE-NEXT: movw %ax, 14(%rsi)
; SSE-NEXT: movw %ax, 12(%rsi)
; SSE-NEXT: movw %ax, 10(%rsi)
; SSE-NEXT: movw %ax, 8(%rsi)
; SSE-NEXT: movw %ax, 6(%rsi)
; SSE-NEXT: movw %ax, 4(%rsi)
; SSE-NEXT: movw %ax, 2(%rsi)
; SSE-NEXT: movw %ax, (%rsi)
; SSE-NEXT: .LBB2_2: # %exit
; SSE-NEXT: retq
;
; AVX-LABEL: splat_v16bf16_cross_block:
; AVX: # %bb.0: # %entry
; AVX-NEXT: testb $1, %dil
; AVX-NEXT: je .LBB2_2
; AVX-NEXT: # %bb.1: # %splat
; AVX-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX-NEXT: vmovdqa %ymm0, (%rsi)
; AVX-NEXT: .LBB2_2: # %exit
; AVX-NEXT: vzeroupper
; AVX-NEXT: retq
entry:
br i1 %c, label %splat, label %exit
splat:
%s = shufflevector <16 x bfloat> %v, <16 x bfloat> poison, <16 x i32> zeroinitializer
store <16 x bfloat> %s, ptr %p
br label %exit
exit:
ret void
}
;; A volatile load cannot be narrowed into a 16-bit broadcast load, so this
;; needs a register broadcast even within a single block.
define void @splat_v8bf16_volatile_load(ptr %src, ptr %dst) nounwind {
; SSE-LABEL: splat_v8bf16_volatile_load:
; SSE: # %bb.0:
; SSE-NEXT: movzwl 14(%rdi), %eax
; SSE-NEXT: movzwl 12(%rdi), %eax
; SSE-NEXT: movzwl 10(%rdi), %eax
; SSE-NEXT: movzwl 8(%rdi), %eax
; SSE-NEXT: movzwl 6(%rdi), %eax
; SSE-NEXT: movzwl 4(%rdi), %eax
; SSE-NEXT: movzwl (%rdi), %eax
; SSE-NEXT: movzwl 2(%rdi), %ecx
; SSE-NEXT: movw %ax, 14(%rsi)
; SSE-NEXT: movw %ax, 12(%rsi)
; SSE-NEXT: movw %ax, 10(%rsi)
; SSE-NEXT: movw %ax, 8(%rsi)
; SSE-NEXT: movw %ax, 6(%rsi)
; SSE-NEXT: movw %ax, 4(%rsi)
; SSE-NEXT: movw %ax, 2(%rsi)
; SSE-NEXT: movw %ax, (%rsi)
; SSE-NEXT: retq
;
; AVX-LABEL: splat_v8bf16_volatile_load:
; AVX: # %bb.0:
; AVX-NEXT: vmovdqa (%rdi), %xmm0
; AVX-NEXT: vpbroadcastw %xmm0, %xmm0
; AVX-NEXT: vmovdqa %xmm0, (%rsi)
; AVX-NEXT: retq
%v = load volatile <8 x bfloat>, ptr %src
%s = shufflevector <8 x bfloat> %v, <8 x bfloat> poison, <8 x i32> zeroinitializer
store <8 x bfloat> %s, ptr %dst
ret void
}
;; A load whose other uses are simplified away after legalization is folded
;; into a bf16 VBROADCAST_LOAD.
define void @splat_v8bf16_broadcast_load(ptr %p, <8 x bfloat> %other, ptr %dst1, ptr %dst2) nounwind {
; SSE-LABEL: splat_v8bf16_broadcast_load:
; SSE: # %bb.0:
; SSE-NEXT: movq %xmm0, %rax
; SSE-NEXT: movq %rax, %rcx
; SSE-NEXT: movq %rax, %r8
; SSE-NEXT: movzwl (%rdi), %edi
; SSE-NEXT: movw %di, 8(%rsi)
; SSE-NEXT: movw %di, (%rsi)
; SSE-NEXT: movw %di, 14(%rsi)
; SSE-NEXT: movw %di, 12(%rsi)
; SSE-NEXT: movw %di, 6(%rsi)
; SSE-NEXT: movw %di, 4(%rsi)
; SSE-NEXT: movw %di, 10(%rsi)
; SSE-NEXT: movw %di, 2(%rsi)
; SSE-NEXT: movw %ax, 8(%rdx)
; SSE-NEXT: movw %ax, (%rdx)
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: shrl $16, %eax
; SSE-NEXT: shrq $32, %rcx
; SSE-NEXT: shrq $48, %r8
; SSE-NEXT: movw %r8w, 14(%rdx)
; SSE-NEXT: movw %cx, 12(%rdx)
; SSE-NEXT: movw %r8w, 6(%rdx)
; SSE-NEXT: movw %cx, 4(%rdx)
; SSE-NEXT: movw %ax, 10(%rdx)
; SSE-NEXT: movw %ax, 2(%rdx)
; SSE-NEXT: retq
;
; AVX-LABEL: splat_v8bf16_broadcast_load:
; AVX: # %bb.0:
; AVX-NEXT: vpbroadcastw (%rdi), %xmm1
; AVX-NEXT: vmovdqa %xmm1, (%rsi)
; AVX-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0]
; AVX-NEXT: vmovaps %xmm0, (%rdx)
; AVX-NEXT: retq
%ld = load <8 x bfloat>, ptr %p
%shuf = shufflevector <8 x bfloat> %ld, <8 x bfloat> %other, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
%splat0 = shufflevector <8 x bfloat> %shuf, <8 x bfloat> poison, <8 x i32> zeroinitializer
store <8 x bfloat> %splat0, ptr %dst1
%other_lo = shufflevector <8 x bfloat> %shuf, <8 x bfloat> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
store <8 x bfloat> %other_lo, ptr %dst2
ret void
}
define void @splat_v16bf16_broadcast_load(ptr %p, <8 x bfloat> %other, ptr %dst1, ptr %dst2) nounwind {
; SSE-LABEL: splat_v16bf16_broadcast_load:
; SSE: # %bb.0:
; SSE-NEXT: movq %xmm0, %rax
; SSE-NEXT: movq %rax, %rcx
; SSE-NEXT: movq %rax, %r8
; SSE-NEXT: movzwl (%rdi), %edi
; SSE-NEXT: movw %di, 28(%rsi)
; SSE-NEXT: movw %di, 26(%rsi)
; SSE-NEXT: movw %di, 24(%rsi)
; SSE-NEXT: movw %di, 22(%rsi)
; SSE-NEXT: movw %di, 20(%rsi)
; SSE-NEXT: movw %di, 18(%rsi)
; SSE-NEXT: movw %di, 16(%rsi)
; SSE-NEXT: movw %di, 30(%rsi)
; SSE-NEXT: movw %ax, 8(%rdx)
; SSE-NEXT: movw %ax, (%rdx)
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: shrl $16, %eax
; SSE-NEXT: shrq $32, %rcx
; SSE-NEXT: shrq $48, %r8
; SSE-NEXT: movw %r8w, 14(%rdx)
; SSE-NEXT: movw %cx, 12(%rdx)
; SSE-NEXT: movw %r8w, 6(%rdx)
; SSE-NEXT: movw %cx, 4(%rdx)
; SSE-NEXT: movw %ax, 10(%rdx)
; SSE-NEXT: movw %ax, 2(%rdx)
; SSE-NEXT: retq
;
; AVX-LABEL: splat_v16bf16_broadcast_load:
; AVX: # %bb.0:
; AVX-NEXT: vpbroadcastw (%rdi), %ymm1
; AVX-NEXT: vmovdqa %ymm1, (%rsi)
; AVX-NEXT: vmovddup {{.*#+}} xmm0 = xmm0[0,0]
; AVX-NEXT: vmovaps %xmm0, (%rdx)
; AVX-NEXT: vzeroupper
; AVX-NEXT: retq
%ld = load <8 x bfloat>, ptr %p
%shuf = shufflevector <8 x bfloat> %ld, <8 x bfloat> %other, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
%splat0 = shufflevector <8 x bfloat> %shuf, <8 x bfloat> poison, <8 x i32> zeroinitializer
%wide = shufflevector <8 x bfloat> poison, <8 x bfloat> %splat0, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
store <16 x bfloat> %wide, ptr %dst1
%other_lo = shufflevector <8 x bfloat> %shuf, <8 x bfloat> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
store <8 x bfloat> %other_lo, ptr %dst2
ret void
}
;; combineTargetShuffle must not look through a bitcast from v8bf16 to form a
;; broadcast of an illegal bf16 vector type (v16bf16 without AVX-NE-CONVERT or
;; AVX512BW).
define void @splat_v16i16_from_bitcast_v8bf16(<8 x bfloat> %v, i1 %c, ptr %dst) nounwind {
; SSE-LABEL: splat_v16i16_from_bitcast_v8bf16:
; SSE: # %bb.0: # %entry
; SSE-NEXT: testb $1, %dil
; SSE-NEXT: je .LBB6_2
; SSE-NEXT: # %bb.1: # %splat
; SSE-NEXT: pslld $16, %xmm0
; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7]
; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
; SSE-NEXT: movdqa %xmm0, 16(%rsi)
; SSE-NEXT: movdqa %xmm0, (%rsi)
; SSE-NEXT: .LBB6_2: # %exit
; SSE-NEXT: retq
;
; AVX-LABEL: splat_v16i16_from_bitcast_v8bf16:
; AVX: # %bb.0: # %entry
; AVX-NEXT: testb $1, %dil
; AVX-NEXT: je .LBB6_2
; AVX-NEXT: # %bb.1: # %splat
; AVX-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX-NEXT: vmovdqa %ymm0, (%rsi)
; AVX-NEXT: .LBB6_2: # %exit
; AVX-NEXT: vzeroupper
; AVX-NEXT: retq
entry:
br i1 %c, label %splat, label %exit
splat:
%bc = bitcast <8 x bfloat> %v to <8 x i16>
%s = shufflevector <8 x i16> %bc, <8 x i16> poison, <16 x i32> zeroinitializer
store <16 x i16> %s, ptr %dst
br label %exit
exit:
ret void
}
define void @rebroadcast_v8i32_from_v8bf16(ptr %src, ptr %dst1, ptr %dst2) nounwind {
; SSE-LABEL: rebroadcast_v8i32_from_v8bf16:
; SSE: # %bb.0:
; SSE-NEXT: movzwl 14(%rdi), %eax
; SSE-NEXT: movzwl 12(%rdi), %eax
; SSE-NEXT: movzwl 10(%rdi), %eax
; SSE-NEXT: movzwl 8(%rdi), %eax
; SSE-NEXT: movzwl 6(%rdi), %eax
; SSE-NEXT: movzwl 2(%rdi), %eax
; SSE-NEXT: movzwl 4(%rdi), %eax
; SSE-NEXT: movzwl (%rdi), %eax
; SSE-NEXT: movl %eax, %ecx
; SSE-NEXT: shll $16, %ecx
; SSE-NEXT: orl %eax, %ecx
; SSE-NEXT: movw %ax, 14(%rsi)
; SSE-NEXT: movw %ax, 12(%rsi)
; SSE-NEXT: movw %ax, 10(%rsi)
; SSE-NEXT: movw %ax, 8(%rsi)
; SSE-NEXT: movw %ax, 6(%rsi)
; SSE-NEXT: movw %ax, 4(%rsi)
; SSE-NEXT: movw %ax, 2(%rsi)
; SSE-NEXT: movw %ax, (%rsi)
; SSE-NEXT: movd %ecx, %xmm0
; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
; SSE-NEXT: movdqa %xmm0, 16(%rdx)
; SSE-NEXT: movdqa %xmm0, (%rdx)
; SSE-NEXT: retq
;
; AVX-LABEL: rebroadcast_v8i32_from_v8bf16:
; AVX: # %bb.0:
; AVX-NEXT: vmovdqa (%rdi), %xmm0
; AVX-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX-NEXT: vmovdqa %xmm0, (%rsi)
; AVX-NEXT: vmovdqa %ymm0, (%rdx)
; AVX-NEXT: vzeroupper
; AVX-NEXT: retq
%v = load volatile <8 x bfloat>, ptr %src
%s8 = shufflevector <8 x bfloat> %v, <8 x bfloat> poison, <8 x i32> zeroinitializer
store <8 x bfloat> %s8, ptr %dst1
%bc = bitcast <8 x bfloat> %s8 to <4 x i32>
%s256 = shufflevector <4 x i32> %bc, <4 x i32> poison, <8 x i32> zeroinitializer
store <8 x i32> %s256, ptr %dst2
ret void
}