blob: 8b13c3e7e666a7c042192edb22f58113f5b0c39d [file] [edit]
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-SSE-O3,CHECK-SSE2-O3
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-SSE-O3,CHECK-SSE4-O3
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-AVX-O3,CHECK-AVX2-O3
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-AVX-O3,CHECK-AVX512-O3
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-SSE-O0,CHECK-SSE2-O0
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-SSE-O0,CHECK-SSE4-O0
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-AVX-O0,CHECK-AVX2-O0
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-AVX-O0,CHECK-AVX512-O0
define void @test1(ptr %ptr, i32 %val1) {
; CHECK-LABEL: test1:
; CHECK: # %bb.0:
; CHECK-NEXT: xchgl %esi, (%rdi)
; CHECK-NEXT: retq
store atomic i32 %val1, ptr %ptr seq_cst, align 4
ret void
}
define void @test2(ptr %ptr, i32 %val1) {
; CHECK-LABEL: test2:
; CHECK: # %bb.0:
; CHECK-NEXT: movl %esi, (%rdi)
; CHECK-NEXT: retq
store atomic i32 %val1, ptr %ptr release, align 4
ret void
}
define i32 @test3(ptr %ptr) {
; CHECK-LABEL: test3:
; CHECK: # %bb.0:
; CHECK-NEXT: movl (%rdi), %eax
; CHECK-NEXT: retq
%val = load atomic i32, ptr %ptr seq_cst, align 4
ret i32 %val
}
define <1 x i32> @atomic_vec1_i32(ptr %x) {
; CHECK-LABEL: atomic_vec1_i32:
; CHECK: # %bb.0:
; CHECK-NEXT: movl (%rdi), %eax
; CHECK-NEXT: retq
%ret = load atomic <1 x i32>, ptr %x acquire, align 4
ret <1 x i32> %ret
}
define <1 x i8> @atomic_vec1_i8(ptr %x) {
; CHECK-O3-LABEL: atomic_vec1_i8:
; CHECK-O3: # %bb.0:
; CHECK-O3-NEXT: movzbl (%rdi), %eax
; CHECK-O3-NEXT: retq
;
; CHECK-O0-LABEL: atomic_vec1_i8:
; CHECK-O0: # %bb.0:
; CHECK-O0-NEXT: movb (%rdi), %al
; CHECK-O0-NEXT: retq
%ret = load atomic <1 x i8>, ptr %x acquire, align 1
ret <1 x i8> %ret
}
define <1 x i16> @atomic_vec1_i16(ptr %x) {
; CHECK-O3-LABEL: atomic_vec1_i16:
; CHECK-O3: # %bb.0:
; CHECK-O3-NEXT: movzwl (%rdi), %eax
; CHECK-O3-NEXT: retq
;
; CHECK-O0-LABEL: atomic_vec1_i16:
; CHECK-O0: # %bb.0:
; CHECK-O0-NEXT: movw (%rdi), %ax
; CHECK-O0-NEXT: retq
%ret = load atomic <1 x i16>, ptr %x acquire, align 2
ret <1 x i16> %ret
}
define <1 x i32> @atomic_vec1_i8_zext(ptr %x) {
; CHECK-O3-LABEL: atomic_vec1_i8_zext:
; CHECK-O3: # %bb.0:
; CHECK-O3-NEXT: movzbl (%rdi), %eax
; CHECK-O3-NEXT: movzbl %al, %eax
; CHECK-O3-NEXT: retq
;
; CHECK-O0-LABEL: atomic_vec1_i8_zext:
; CHECK-O0: # %bb.0:
; CHECK-O0-NEXT: movb (%rdi), %al
; CHECK-O0-NEXT: movzbl %al, %eax
; CHECK-O0-NEXT: retq
%ret = load atomic <1 x i8>, ptr %x acquire, align 1
%zret = zext <1 x i8> %ret to <1 x i32>
ret <1 x i32> %zret
}
define <1 x i64> @atomic_vec1_i16_sext(ptr %x) {
; CHECK-O3-LABEL: atomic_vec1_i16_sext:
; CHECK-O3: # %bb.0:
; CHECK-O3-NEXT: movzwl (%rdi), %eax
; CHECK-O3-NEXT: movswq %ax, %rax
; CHECK-O3-NEXT: retq
;
; CHECK-O0-LABEL: atomic_vec1_i16_sext:
; CHECK-O0: # %bb.0:
; CHECK-O0-NEXT: movw (%rdi), %ax
; CHECK-O0-NEXT: movswq %ax, %rax
; CHECK-O0-NEXT: retq
%ret = load atomic <1 x i16>, ptr %x acquire, align 2
%sret = sext <1 x i16> %ret to <1 x i64>
ret <1 x i64> %sret
}
define <1 x ptr addrspace(270)> @atomic_vec1_ptr270(ptr %x) {
; CHECK-LABEL: atomic_vec1_ptr270:
; CHECK: # %bb.0:
; CHECK-NEXT: movl (%rdi), %eax
; CHECK-NEXT: retq
%ret = load atomic <1 x ptr addrspace(270)>, ptr %x acquire, align 4
ret <1 x ptr addrspace(270)> %ret
}
define <1 x bfloat> @atomic_vec1_bfloat(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec1_bfloat:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movzwl (%rdi), %eax
; CHECK-SSE-O3-NEXT: movd %eax, %xmm0
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_bfloat:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: movzwl (%rdi), %eax
; CHECK-AVX-O3-NEXT: vmovd %eax, %xmm0
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_bfloat:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movw (%rdi), %cx
; CHECK-SSE-O0-NEXT: # implicit-def: $eax
; CHECK-SSE-O0-NEXT: movw %cx, %ax
; CHECK-SSE-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_bfloat:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: movw (%rdi), %cx
; CHECK-AVX-O0-NEXT: # implicit-def: $eax
; CHECK-AVX-O0-NEXT: movw %cx, %ax
; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <1 x bfloat>, ptr %x acquire, align 2
ret <1 x bfloat> %ret
}
define <1 x ptr> @atomic_vec1_ptr_align(ptr %x) nounwind {
; CHECK-LABEL: atomic_vec1_ptr_align:
; CHECK: # %bb.0:
; CHECK-NEXT: movq (%rdi), %rax
; CHECK-NEXT: retq
%ret = load atomic <1 x ptr>, ptr %x acquire, align 8
ret <1 x ptr> %ret
}
define <1 x i64> @atomic_vec1_i64_align(ptr %x) nounwind {
; CHECK-LABEL: atomic_vec1_i64_align:
; CHECK: # %bb.0:
; CHECK-NEXT: movq (%rdi), %rax
; CHECK-NEXT: retq
%ret = load atomic <1 x i64>, ptr %x acquire, align 8
ret <1 x i64> %ret
}
define void @store_atomic_vec1_i32(ptr %x, <1 x i32> %v) {
; CHECK-LABEL: store_atomic_vec1_i32:
; CHECK: # %bb.0:
; CHECK-NEXT: movl %esi, (%rdi)
; CHECK-NEXT: retq
store atomic <1 x i32> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec1_i8(ptr %x, <1 x i8> %v) {
; CHECK-O3-LABEL: store_atomic_vec1_i8:
; CHECK-O3: # %bb.0:
; CHECK-O3-NEXT: movb %sil, (%rdi)
; CHECK-O3-NEXT: retq
;
; CHECK-O0-LABEL: store_atomic_vec1_i8:
; CHECK-O0: # %bb.0:
; CHECK-O0-NEXT: movb %sil, %al
; CHECK-O0-NEXT: movb %al, (%rdi)
; CHECK-O0-NEXT: retq
store atomic <1 x i8> %v, ptr %x release, align 1
ret void
}
define void @store_atomic_vec1_i16(ptr %x, <1 x i16> %v) {
; CHECK-O3-LABEL: store_atomic_vec1_i16:
; CHECK-O3: # %bb.0:
; CHECK-O3-NEXT: movw %si, (%rdi)
; CHECK-O3-NEXT: retq
;
; CHECK-O0-LABEL: store_atomic_vec1_i16:
; CHECK-O0: # %bb.0:
; CHECK-O0-NEXT: movw %si, %ax
; CHECK-O0-NEXT: movw %ax, (%rdi)
; CHECK-O0-NEXT: retq
store atomic <1 x i16> %v, ptr %x release, align 2
ret void
}
define void @store_atomic_vec1_i64(ptr %x, <1 x i64> %v) nounwind {
; CHECK-LABEL: store_atomic_vec1_i64:
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rsi, (%rdi)
; CHECK-NEXT: retq
store atomic <1 x i64> %v, ptr %x release, align 8
ret void
}
define void @store_atomic_vec1_ptr(ptr %x, <1 x ptr> %v) nounwind {
; CHECK-LABEL: store_atomic_vec1_ptr:
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rsi, (%rdi)
; CHECK-NEXT: retq
store atomic <1 x ptr> %v, ptr %x release, align 8
ret void
}
define void @store_atomic_vec1_bfloat(ptr %x, <1 x bfloat> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_bfloat:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: pextrw $0, %xmm0, %eax
; CHECK-SSE-O3-NEXT: movw %ax, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_bfloat:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vpextrw $0, %xmm0, %eax
; CHECK-AVX-O3-NEXT: movw %ax, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_bfloat:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: pslld $16, %xmm0
; CHECK-SSE-O0-NEXT: movd %xmm0, %ecx
; CHECK-SSE-O0-NEXT: btl $16, %ecx
; CHECK-SSE-O0-NEXT: movl %ecx, %eax
; CHECK-SSE-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
; CHECK-SSE-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
; CHECK-SSE-O0-NEXT: ucomiss %xmm0, %xmm0
; CHECK-SSE-O0-NEXT: cmovpl %ecx, %eax
; CHECK-SSE-O0-NEXT: shrl $16, %eax
; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE-O0-NEXT: movw %ax, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_bfloat:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vpslld $16, %xmm0, %xmm0
; CHECK-AVX-O0-NEXT: vmovd %xmm0, %ecx
; CHECK-AVX-O0-NEXT: btl $16, %ecx
; CHECK-AVX-O0-NEXT: movl %ecx, %eax
; CHECK-AVX-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF
; CHECK-AVX-O0-NEXT: orl $4194304, %ecx # imm = 0x400000
; CHECK-AVX-O0-NEXT: vucomiss %xmm0, %xmm0
; CHECK-AVX-O0-NEXT: cmovpl %ecx, %eax
; CHECK-AVX-O0-NEXT: shrl $16, %eax
; CHECK-AVX-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-AVX-O0-NEXT: movw %ax, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <1 x bfloat> %v, ptr %x release, align 2
ret void
}
define void @store_atomic_vec1_half(ptr %x, <1 x half> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_half:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: pextrw $0, %xmm0, %eax
; CHECK-SSE-O3-NEXT: movw %ax, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_half:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vpextrw $0, %xmm0, %eax
; CHECK-AVX-O3-NEXT: movw %ax, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_half:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: pextrw $0, %xmm0, %eax
; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE-O0-NEXT: movw %ax, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_half:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, %eax
; CHECK-AVX-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-AVX-O0-NEXT: movw %ax, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <1 x half> %v, ptr %x release, align 2
ret void
}
define void @store_atomic_vec1_float(ptr %x, <1 x float> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_float:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_float:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_float:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movss %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_float:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovss %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <1 x float> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec1_double_align(ptr %x, <1 x double> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_double_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movsd %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_double_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovsd %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_double_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movsd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_double_align:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovsd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <1 x double> %v, ptr %x release, align 8
ret void
}
define void @store_atomic_vec2_i8(ptr %x, <2 x i8> %v) {
; CHECK-SSE2-O3-LABEL: store_atomic_vec2_i8:
; CHECK-SSE2-O3: # %bb.0:
; CHECK-SSE2-O3-NEXT: movd %xmm0, %eax
; CHECK-SSE2-O3-NEXT: movw %ax, (%rdi)
; CHECK-SSE2-O3-NEXT: retq
;
; CHECK-SSE4-O3-LABEL: store_atomic_vec2_i8:
; CHECK-SSE4-O3: # %bb.0:
; CHECK-SSE4-O3-NEXT: pextrw $0, %xmm0, (%rdi)
; CHECK-SSE4-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i8:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vpextrw $0, %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec2_i8:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, (%rdi)
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec2_i8:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm0, (%rdi)
; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i8:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x i8> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec2_i16(ptr %x, <2 x i16> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_i16:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i16:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_i16:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i16:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x i16> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec2_ptr270(ptr %x, <2 x ptr addrspace(270)> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_ptr270:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_ptr270:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_ptr270:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_ptr270:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x ptr addrspace(270)> %v, ptr %x release, align 8
ret void
}
define void @store_atomic_vec2_i32_align(ptr %x, <2 x i32> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_i32_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i32_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_i32_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i32_align:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x i32> %v, ptr %x release, align 8
ret void
}
define void @store_atomic_vec2_float_align(ptr %x, <2 x float> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_float_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_float_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_float_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_float_align:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x float> %v, ptr %x release, align 8
ret void
}
define void @store_atomic_vec4_i8(ptr %x, <4 x i8> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_i8:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_i8:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec4_i8:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_i8:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <4 x i8> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec4_i16(ptr %x, <4 x i16> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_i16:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_i16:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec4_i16:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_i16:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <4 x i16> %v, ptr %x release, align 8
ret void
}
define <2 x i8> @atomic_vec2_i8(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_i8:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movzwl (%rdi), %eax
; CHECK-SSE-O3-NEXT: movd %eax, %xmm0
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_i8:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: movzwl (%rdi), %eax
; CHECK-AVX-O3-NEXT: vmovd %eax, %xmm0
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_i8:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movw (%rdi), %cx
; CHECK-SSE-O0-NEXT: # implicit-def: $eax
; CHECK-SSE-O0-NEXT: movw %cx, %ax
; CHECK-SSE-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_i8:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: movw (%rdi), %cx
; CHECK-AVX-O0-NEXT: # implicit-def: $eax
; CHECK-AVX-O0-NEXT: movw %cx, %ax
; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <2 x i8>, ptr %x acquire, align 4
ret <2 x i8> %ret
}
define <2 x i16> @atomic_vec2_i16(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_i16:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_i16:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_i16:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_i16:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <2 x i16>, ptr %x acquire, align 4
ret <2 x i16> %ret
}
define <2 x ptr addrspace(270)> @atomic_vec2_ptr270(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_ptr270:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_ptr270:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_ptr270:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_ptr270:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <2 x ptr addrspace(270)>, ptr %x acquire, align 8
ret <2 x ptr addrspace(270)> %ret
}
define <2 x ptr> @atomic_vec2_ptr_align(ptr %x) nounwind {
; CHECK-SSE2-O3-LABEL: atomic_vec2_ptr_align:
; CHECK-SSE2-O3: # %bb.0:
; CHECK-SSE2-O3-NEXT: pushq %rax
; CHECK-SSE2-O3-NEXT: movl $2, %esi
; CHECK-SSE2-O3-NEXT: callq __atomic_load_16@PLT
; CHECK-SSE2-O3-NEXT: movq %rdx, %xmm1
; CHECK-SSE2-O3-NEXT: movq %rax, %xmm0
; CHECK-SSE2-O3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O3-NEXT: popq %rax
; CHECK-SSE2-O3-NEXT: retq
;
; CHECK-SSE4-O3-LABEL: atomic_vec2_ptr_align:
; CHECK-SSE4-O3: # %bb.0:
; CHECK-SSE4-O3-NEXT: movaps (%rdi), %xmm0
; CHECK-SSE4-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_ptr_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovaps (%rdi), %xmm0
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE2-O0-LABEL: atomic_vec2_ptr_align:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: pushq %rax
; CHECK-SSE2-O0-NEXT: movl $2, %esi
; CHECK-SSE2-O0-NEXT: callq __atomic_load_16@PLT
; CHECK-SSE2-O0-NEXT: movq %rdx, %xmm1
; CHECK-SSE2-O0-NEXT: movq %rax, %xmm0
; CHECK-SSE2-O0-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O0-NEXT: popq %rax
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: atomic_vec2_ptr_align:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: movapd (%rdi), %xmm0
; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_ptr_align:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovapd (%rdi), %xmm0
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <2 x ptr>, ptr %x acquire, align 16
ret <2 x ptr> %ret
}
define void @store_atomic_vec2_ptr_align(ptr %x, <2 x ptr> %v) nounwind {
; CHECK-SSE2-O3-LABEL: store_atomic_vec2_ptr_align:
; CHECK-SSE2-O3: # %bb.0:
; CHECK-SSE2-O3-NEXT: pushq %rax
; CHECK-SSE2-O3-NEXT: movq %xmm0, %rsi
; CHECK-SSE2-O3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; CHECK-SSE2-O3-NEXT: movq %xmm0, %rdx
; CHECK-SSE2-O3-NEXT: movl $3, %ecx
; CHECK-SSE2-O3-NEXT: callq __atomic_store_16@PLT
; CHECK-SSE2-O3-NEXT: popq %rax
; CHECK-SSE2-O3-NEXT: retq
;
; CHECK-SSE4-O3-LABEL: store_atomic_vec2_ptr_align:
; CHECK-SSE4-O3: # %bb.0:
; CHECK-SSE4-O3-NEXT: movaps %xmm0, (%rdi)
; CHECK-SSE4-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_ptr_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovaps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec2_ptr_align:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: pushq %rax
; CHECK-SSE2-O0-NEXT: movq %xmm0, %rsi
; CHECK-SSE2-O0-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; CHECK-SSE2-O0-NEXT: movq %xmm0, %rdx
; CHECK-SSE2-O0-NEXT: movl $3, %ecx
; CHECK-SSE2-O0-NEXT: callq __atomic_store_16@PLT
; CHECK-SSE2-O0-NEXT: popq %rax
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec2_ptr_align:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: movaps %xmm0, (%rdi)
; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_ptr_align:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovdqa %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x ptr> %v, ptr %x release, align 16
ret void
}
define <4 x ptr addrspace(270)> @atomic_vec4_ptr270(ptr %x) nounwind {
; CHECK-SSE2-O3-LABEL: atomic_vec4_ptr270:
; CHECK-SSE2-O3: # %bb.0:
; CHECK-SSE2-O3-NEXT: pushq %rax
; CHECK-SSE2-O3-NEXT: movl $2, %esi
; CHECK-SSE2-O3-NEXT: callq __atomic_load_16@PLT
; CHECK-SSE2-O3-NEXT: movq %rdx, %xmm1
; CHECK-SSE2-O3-NEXT: movq %rax, %xmm0
; CHECK-SSE2-O3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O3-NEXT: popq %rax
; CHECK-SSE2-O3-NEXT: retq
;
; CHECK-SSE4-O3-LABEL: atomic_vec4_ptr270:
; CHECK-SSE4-O3: # %bb.0:
; CHECK-SSE4-O3-NEXT: movaps (%rdi), %xmm0
; CHECK-SSE4-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_ptr270:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovaps (%rdi), %xmm0
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE2-O0-LABEL: atomic_vec4_ptr270:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: pushq %rax
; CHECK-SSE2-O0-NEXT: movl $2, %esi
; CHECK-SSE2-O0-NEXT: callq __atomic_load_16@PLT
; CHECK-SSE2-O0-NEXT: movq %rdx, %xmm1
; CHECK-SSE2-O0-NEXT: movq %rax, %xmm0
; CHECK-SSE2-O0-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O0-NEXT: popq %rax
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: atomic_vec4_ptr270:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: movapd (%rdi), %xmm0
; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_ptr270:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovapd (%rdi), %xmm0
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <4 x ptr addrspace(270)>, ptr %x acquire, align 16
ret <4 x ptr addrspace(270)> %ret
}
define void @store_atomic_vec4_ptr270_align(ptr %x, <4 x ptr addrspace(270)> %v) nounwind {
; CHECK-SSE2-O3-LABEL: store_atomic_vec4_ptr270_align:
; CHECK-SSE2-O3: # %bb.0:
; CHECK-SSE2-O3-NEXT: pushq %rax
; CHECK-SSE2-O3-NEXT: movq %xmm0, %rsi
; CHECK-SSE2-O3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; CHECK-SSE2-O3-NEXT: movq %xmm0, %rdx
; CHECK-SSE2-O3-NEXT: movl $3, %ecx
; CHECK-SSE2-O3-NEXT: callq __atomic_store_16@PLT
; CHECK-SSE2-O3-NEXT: popq %rax
; CHECK-SSE2-O3-NEXT: retq
;
; CHECK-SSE4-O3-LABEL: store_atomic_vec4_ptr270_align:
; CHECK-SSE4-O3: # %bb.0:
; CHECK-SSE4-O3-NEXT: movaps %xmm0, (%rdi)
; CHECK-SSE4-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_ptr270_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovaps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec4_ptr270_align:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: pushq %rax
; CHECK-SSE2-O0-NEXT: movq %xmm0, %rsi
; CHECK-SSE2-O0-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; CHECK-SSE2-O0-NEXT: movq %xmm0, %rdx
; CHECK-SSE2-O0-NEXT: movl $3, %ecx
; CHECK-SSE2-O0-NEXT: callq __atomic_store_16@PLT
; CHECK-SSE2-O0-NEXT: popq %rax
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec4_ptr270_align:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: movaps %xmm0, (%rdi)
; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_ptr270_align:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovdqa %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <4 x ptr addrspace(270)> %v, ptr %x release, align 16
ret void
}
define <2 x i32> @atomic_vec2_i32_align(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_i32_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_i32_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_i32_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_i32_align:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <2 x i32>, ptr %x acquire, align 8
ret <2 x i32> %ret
}
define <2 x float> @atomic_vec2_float_align(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_float_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_float_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_float_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_float_align:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <2 x float>, ptr %x acquire, align 8
ret <2 x float> %ret
}
define void @store_atomic_vec2_half(ptr %x, <2 x half> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_half:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_half:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_half:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE-O0-NEXT: movaps %xmm1, %xmm0
; CHECK-SSE-O0-NEXT: psrld $16, %xmm1
; CHECK-SSE-O0-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; CHECK-SSE-O0-NEXT: movd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_half:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x half> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec2_bfloat(ptr %x, <2 x bfloat> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_bfloat:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_bfloat:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec2_bfloat:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm0, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
; CHECK-SSE2-O0-NEXT: movd %xmm1, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movl -{{[0-9]+}}(%rsp), %eax
; CHECK-SSE2-O0-NEXT: movl %eax, (%rdi)
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec2_bfloat:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT: xorps %xmm2, %xmm2
; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm0
; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7]
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
; CHECK-SSE4-O0-NEXT: movd %xmm1, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm0, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movl -{{[0-9]+}}(%rsp), %eax
; CHECK-SSE4-O0-NEXT: movl %eax, (%rdi)
; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_bfloat:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x bfloat> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec4_half(ptr %x, <4 x half> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_half:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_half:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec4_half:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm3
; CHECK-SSE2-O0-NEXT: movaps %xmm3, %xmm0
; CHECK-SSE2-O0-NEXT: movaps %xmm3, %xmm2
; CHECK-SSE2-O0-NEXT: psrlq $48, %xmm2
; CHECK-SSE2-O0-NEXT: movaps %xmm3, %xmm1
; CHECK-SSE2-O0-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
; CHECK-SSE2-O0-NEXT: psrld $16, %xmm3
; CHECK-SSE2-O0-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
; CHECK-SSE2-O0-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
; CHECK-SSE2-O0-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; CHECK-SSE2-O0-NEXT: movq %xmm0, (%rdi)
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec4_half:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm3
; CHECK-SSE4-O0-NEXT: movaps %xmm3, %xmm0
; CHECK-SSE4-O0-NEXT: movaps %xmm3, %xmm2
; CHECK-SSE4-O0-NEXT: psrlq $48, %xmm2
; CHECK-SSE4-O0-NEXT: movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
; CHECK-SSE4-O0-NEXT: psrld $16, %xmm3
; CHECK-SSE4-O0-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
; CHECK-SSE4-O0-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
; CHECK-SSE4-O0-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; CHECK-SSE4-O0-NEXT: movq %xmm0, (%rdi)
; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_half:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <4 x half> %v, ptr %x release, align 8
ret void
}
define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_bfloat:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm2
; CHECK-SSE2-O0-NEXT: movaps %xmm2, %xmm3
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm3
; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm2, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm2, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm2, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
; CHECK-SSE2-O0-NEXT: movd %xmm3, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm2, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm1, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movq -{{[0-9]+}}(%rsp), %rax
; CHECK-SSE2-O0-NEXT: movq %rax, (%rdi)
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0
; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm2
; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm3
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm3
; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm1, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm1, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
; CHECK-SSE4-O0-NEXT: movd %xmm3, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm2, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm1, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm0, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movq -{{[0-9]+}}(%rsp), %rax
; CHECK-SSE4-O0-NEXT: movq %rax, (%rdi)
; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <4 x bfloat> %v, ptr %x release, align 8
ret void
}
define void @store_atomic_vec4_float_align(ptr %x, <4 x float> %v) nounwind {
; CHECK-SSE2-O3-LABEL: store_atomic_vec4_float_align:
; CHECK-SSE2-O3: # %bb.0:
; CHECK-SSE2-O3-NEXT: pushq %rax
; CHECK-SSE2-O3-NEXT: movq %xmm0, %rsi
; CHECK-SSE2-O3-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-SSE2-O3-NEXT: movq %xmm0, %rdx
; CHECK-SSE2-O3-NEXT: movl $3, %ecx
; CHECK-SSE2-O3-NEXT: callq __atomic_store_16@PLT
; CHECK-SSE2-O3-NEXT: popq %rax
; CHECK-SSE2-O3-NEXT: retq
;
; CHECK-SSE4-O3-LABEL: store_atomic_vec4_float_align:
; CHECK-SSE4-O3: # %bb.0:
; CHECK-SSE4-O3-NEXT: pushq %rbx
; CHECK-SSE4-O3-NEXT: movdqa (%rdi), %xmm1
; CHECK-SSE4-O3-NEXT: pextrq $1, %xmm0, %rcx
; CHECK-SSE4-O3-NEXT: movq %xmm0, %rbx
; CHECK-SSE4-O3-NEXT: .p2align 4
; CHECK-SSE4-O3-NEXT: .LBB41_1: # %atomicrmw.start
; CHECK-SSE4-O3-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-SSE4-O3-NEXT: movq %xmm1, %rax
; CHECK-SSE4-O3-NEXT: pextrq $1, %xmm1, %rdx
; CHECK-SSE4-O3-NEXT: lock cmpxchg16b (%rdi)
; CHECK-SSE4-O3-NEXT: movq %rdx, %xmm0
; CHECK-SSE4-O3-NEXT: movq %rax, %xmm1
; CHECK-SSE4-O3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
; CHECK-SSE4-O3-NEXT: jne .LBB41_1
; CHECK-SSE4-O3-NEXT: # %bb.2: # %atomicrmw.end
; CHECK-SSE4-O3-NEXT: popq %rbx
; CHECK-SSE4-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_float_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovaps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec4_float_align:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: pushq %rax
; CHECK-SSE2-O0-NEXT: movq %xmm0, %rsi
; CHECK-SSE2-O0-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-SSE2-O0-NEXT: movq %xmm0, %rdx
; CHECK-SSE2-O0-NEXT: movl $3, %ecx
; CHECK-SSE2-O0-NEXT: callq __atomic_store_16@PLT
; CHECK-SSE2-O0-NEXT: popq %rax
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec4_float_align:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: pushq %rbx
; CHECK-SSE4-O0-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE4-O0-NEXT: movaps (%rdi), %xmm0
; CHECK-SSE4-O0-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-SSE4-O0-NEXT: .LBB41_1: # %atomicrmw.start
; CHECK-SSE4-O0-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-SSE4-O0-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; CHECK-SSE4-O0-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; CHECK-SSE4-O0-NEXT: pextrq $1, %xmm1, %rcx
; CHECK-SSE4-O0-NEXT: movq %xmm1, %rbx
; CHECK-SSE4-O0-NEXT: movq %xmm0, %rax
; CHECK-SSE4-O0-NEXT: pextrq $1, %xmm0, %rdx
; CHECK-SSE4-O0-NEXT: lock cmpxchg16b (%rsi)
; CHECK-SSE4-O0-NEXT: movq %rax, %rcx
; CHECK-SSE4-O0-NEXT: sete %al
; CHECK-SSE4-O0-NEXT: movq %rdx, %xmm1
; CHECK-SSE4-O0-NEXT: movq %rcx, %xmm0
; CHECK-SSE4-O0-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE4-O0-NEXT: testb $1, %al
; CHECK-SSE4-O0-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-SSE4-O0-NEXT: jne .LBB41_2
; CHECK-SSE4-O0-NEXT: jmp .LBB41_1
; CHECK-SSE4-O0-NEXT: .LBB41_2: # %atomicrmw.end
; CHECK-SSE4-O0-NEXT: popq %rbx
; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_float_align:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovaps %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <4 x float> %v, ptr %x release, align 16
ret void
}
define <2 x half> @atomic_vec2_half(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_half:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_half:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_half:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_half:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <2 x half>, ptr %x acquire, align 4
ret <2 x half> %ret
}
define <2 x bfloat> @atomic_vec2_bfloat(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_bfloat:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_bfloat:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_bfloat:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_bfloat:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <2 x bfloat>, ptr %x acquire, align 4
ret <2 x bfloat> %ret
}
define <1 x ptr> @atomic_vec1_ptr(ptr %x) nounwind {
; CHECK-O3-LABEL: atomic_vec1_ptr:
; CHECK-O3: # %bb.0:
; CHECK-O3-NEXT: pushq %rax
; CHECK-O3-NEXT: movq %rdi, %rsi
; CHECK-O3-NEXT: movq %rsp, %rdx
; CHECK-O3-NEXT: movl $8, %edi
; CHECK-O3-NEXT: movl $2, %ecx
; CHECK-O3-NEXT: callq __atomic_load@PLT
; CHECK-O3-NEXT: movq (%rsp), %rax
; CHECK-O3-NEXT: popq %rcx
; CHECK-O3-NEXT: retq
;
; CHECK-O0-LABEL: atomic_vec1_ptr:
; CHECK-O0: # %bb.0:
; CHECK-O0-NEXT: pushq %rax
; CHECK-O0-NEXT: movq %rdi, %rsi
; CHECK-O0-NEXT: movl $8, %edi
; CHECK-O0-NEXT: movq %rsp, %rdx
; CHECK-O0-NEXT: movl $2, %ecx
; CHECK-O0-NEXT: callq __atomic_load@PLT
; CHECK-O0-NEXT: movq (%rsp), %rax
; CHECK-O0-NEXT: popq %rcx
; CHECK-O0-NEXT: retq
%ret = load atomic <1 x ptr>, ptr %x acquire, align 4
ret <1 x ptr> %ret
}
define <1 x half> @atomic_vec1_half(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec1_half:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movzwl (%rdi), %eax
; CHECK-SSE-O3-NEXT: movd %eax, %xmm0
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_half:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: movzwl (%rdi), %eax
; CHECK-AVX-O3-NEXT: vmovd %eax, %xmm0
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_half:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movw (%rdi), %cx
; CHECK-SSE-O0-NEXT: # implicit-def: $eax
; CHECK-SSE-O0-NEXT: movw %cx, %ax
; CHECK-SSE-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_half:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: movw (%rdi), %cx
; CHECK-AVX-O0-NEXT: # implicit-def: $eax
; CHECK-AVX-O0-NEXT: movw %cx, %ax
; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <1 x half>, ptr %x acquire, align 2
ret <1 x half> %ret
}
define <1 x float> @atomic_vec1_float(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec1_float:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_float:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_float:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_float:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <1 x float>, ptr %x acquire, align 4
ret <1 x float> %ret
}
define <1 x double> @atomic_vec1_double_align(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec1_double_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_double_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_double_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_double_align:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <1 x double>, ptr %x acquire, align 8
ret <1 x double> %ret
}
define <1 x i64> @atomic_vec1_i64(ptr %x) nounwind {
; CHECK-O3-LABEL: atomic_vec1_i64:
; CHECK-O3: # %bb.0:
; CHECK-O3-NEXT: pushq %rax
; CHECK-O3-NEXT: movq %rdi, %rsi
; CHECK-O3-NEXT: movq %rsp, %rdx
; CHECK-O3-NEXT: movl $8, %edi
; CHECK-O3-NEXT: movl $2, %ecx
; CHECK-O3-NEXT: callq __atomic_load@PLT
; CHECK-O3-NEXT: movq (%rsp), %rax
; CHECK-O3-NEXT: popq %rcx
; CHECK-O3-NEXT: retq
;
; CHECK-O0-LABEL: atomic_vec1_i64:
; CHECK-O0: # %bb.0:
; CHECK-O0-NEXT: pushq %rax
; CHECK-O0-NEXT: movq %rdi, %rsi
; CHECK-O0-NEXT: movl $8, %edi
; CHECK-O0-NEXT: movq %rsp, %rdx
; CHECK-O0-NEXT: movl $2, %ecx
; CHECK-O0-NEXT: callq __atomic_load@PLT
; CHECK-O0-NEXT: movq (%rsp), %rax
; CHECK-O0-NEXT: popq %rcx
; CHECK-O0-NEXT: retq
%ret = load atomic <1 x i64>, ptr %x acquire, align 4
ret <1 x i64> %ret
}
define <1 x double> @atomic_vec1_double(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec1_double:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: pushq %rax
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $8, %edi
; CHECK-SSE-O3-NEXT: movl $2, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O3-NEXT: popq %rax
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_double:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: pushq %rax
; CHECK-AVX-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX-O3-NEXT: movl $8, %edi
; CHECK-AVX-O3-NEXT: movl $2, %ecx
; CHECK-AVX-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O3-NEXT: popq %rax
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_double:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: pushq %rax
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movl $8, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $2, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O0-NEXT: popq %rax
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_double:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: pushq %rax
; CHECK-AVX-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX-O0-NEXT: movl $8, %edi
; CHECK-AVX-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX-O0-NEXT: movl $2, %ecx
; CHECK-AVX-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O0-NEXT: popq %rax
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <1 x double>, ptr %x acquire, align 4
ret <1 x double> %ret
}
define <2 x i32> @atomic_vec2_i32(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec2_i32:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: pushq %rax
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $8, %edi
; CHECK-SSE-O3-NEXT: movl $2, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O3-NEXT: popq %rax
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_i32:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: pushq %rax
; CHECK-AVX-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX-O3-NEXT: movl $8, %edi
; CHECK-AVX-O3-NEXT: movl $2, %ecx
; CHECK-AVX-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O3-NEXT: popq %rax
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_i32:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: pushq %rax
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movl $8, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $2, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O0-NEXT: popq %rax
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_i32:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: pushq %rax
; CHECK-AVX-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX-O0-NEXT: movl $8, %edi
; CHECK-AVX-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX-O0-NEXT: movl $2, %ecx
; CHECK-AVX-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O0-NEXT: popq %rax
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <2 x i32>, ptr %x acquire, align 4
ret <2 x i32> %ret
}
define <4 x i8> @atomic_vec4_i8(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_i8:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_i8:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_i8:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_i8:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <4 x i8>, ptr %x acquire, align 4
ret <4 x i8> %ret
}
define <4 x i16> @atomic_vec4_i16(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_i16:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_i16:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_i16:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_i16:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <4 x i16>, ptr %x acquire, align 8
ret <4 x i16> %ret
}
define <4 x half> @atomic_vec4_half(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_half:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_half:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_half:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_half:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <4 x half>, ptr %x acquire, align 8
ret <4 x half> %ret
}
define <4 x bfloat> @atomic_vec4_bfloat(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_bfloat:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_bfloat:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_bfloat:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_bfloat:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <4 x bfloat>, ptr %x acquire, align 8
ret <4 x bfloat> %ret
}
define <4 x float> @atomic_vec4_float(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_float:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $24, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $16, %edi
; CHECK-SSE-O3-NEXT: movl $2, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT: addq $24, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_float:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: subq $24, %rsp
; CHECK-AVX-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX-O3-NEXT: movl $16, %edi
; CHECK-AVX-O3-NEXT: movl $2, %ecx
; CHECK-AVX-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT: vmovaps (%rsp), %xmm0
; CHECK-AVX-O3-NEXT: addq $24, %rsp
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_float:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $24, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movl $16, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $2, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT: addq $24, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_float:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: subq $24, %rsp
; CHECK-AVX-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX-O0-NEXT: movl $16, %edi
; CHECK-AVX-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX-O0-NEXT: movl $2, %ecx
; CHECK-AVX-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT: vmovaps (%rsp), %xmm0
; CHECK-AVX-O0-NEXT: addq $24, %rsp
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <4 x float>, ptr %x acquire, align 4
ret <4 x float> %ret
}
define <4 x float> @atomic_vec4_float_align(ptr %x) nounwind {
; CHECK-SSE2-O3-LABEL: atomic_vec4_float_align:
; CHECK-SSE2-O3: # %bb.0:
; CHECK-SSE2-O3-NEXT: pushq %rax
; CHECK-SSE2-O3-NEXT: movl $2, %esi
; CHECK-SSE2-O3-NEXT: callq __atomic_load_16@PLT
; CHECK-SSE2-O3-NEXT: movq %rdx, %xmm1
; CHECK-SSE2-O3-NEXT: movq %rax, %xmm0
; CHECK-SSE2-O3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O3-NEXT: popq %rax
; CHECK-SSE2-O3-NEXT: retq
;
; CHECK-SSE4-O3-LABEL: atomic_vec4_float_align:
; CHECK-SSE4-O3: # %bb.0:
; CHECK-SSE4-O3-NEXT: pushq %rbx
; CHECK-SSE4-O3-NEXT: xorl %eax, %eax
; CHECK-SSE4-O3-NEXT: xorl %edx, %edx
; CHECK-SSE4-O3-NEXT: xorl %ecx, %ecx
; CHECK-SSE4-O3-NEXT: xorl %ebx, %ebx
; CHECK-SSE4-O3-NEXT: lock cmpxchg16b (%rdi)
; CHECK-SSE4-O3-NEXT: movq %rdx, %xmm1
; CHECK-SSE4-O3-NEXT: movq %rax, %xmm0
; CHECK-SSE4-O3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE4-O3-NEXT: popq %rbx
; CHECK-SSE4-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_float_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: vmovaps (%rdi), %xmm0
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE2-O0-LABEL: atomic_vec4_float_align:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: pushq %rax
; CHECK-SSE2-O0-NEXT: movl $2, %esi
; CHECK-SSE2-O0-NEXT: callq __atomic_load_16@PLT
; CHECK-SSE2-O0-NEXT: movq %rdx, %xmm1
; CHECK-SSE2-O0-NEXT: movq %rax, %xmm0
; CHECK-SSE2-O0-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O0-NEXT: popq %rax
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: atomic_vec4_float_align:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: pushq %rbx
; CHECK-SSE4-O0-NEXT: xorl %eax, %eax
; CHECK-SSE4-O0-NEXT: movl %eax, %ebx
; CHECK-SSE4-O0-NEXT: movq %rbx, %rax
; CHECK-SSE4-O0-NEXT: movq %rbx, %rdx
; CHECK-SSE4-O0-NEXT: movq %rbx, %rcx
; CHECK-SSE4-O0-NEXT: lock cmpxchg16b (%rdi)
; CHECK-SSE4-O0-NEXT: movq %rdx, %xmm1
; CHECK-SSE4-O0-NEXT: movq %rax, %xmm0
; CHECK-SSE4-O0-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE4-O0-NEXT: popq %rbx
; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_float_align:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: vmovaps (%rdi), %xmm0
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <4 x float>, ptr %x acquire, align 16
ret <4 x float> %ret
}
define <8 x double> @atomic_vec8_double(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec8_double:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $72, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $64, %edi
; CHECK-SSE-O3-NEXT: movl $2, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O3-NEXT: addq $72, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX2-O3-LABEL: atomic_vec8_double:
; CHECK-AVX2-O3: # %bb.0:
; CHECK-AVX2-O3-NEXT: subq $72, %rsp
; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT: movl $64, %edi
; CHECK-AVX2-O3-NEXT: movl $2, %ecx
; CHECK-AVX2-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX2-O3-NEXT: vmovups (%rsp), %ymm0
; CHECK-AVX2-O3-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O3-NEXT: addq $72, %rsp
; CHECK-AVX2-O3-NEXT: retq
;
; CHECK-AVX512-O3-LABEL: atomic_vec8_double:
; CHECK-AVX512-O3: # %bb.0:
; CHECK-AVX512-O3-NEXT: subq $72, %rsp
; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT: movl $64, %edi
; CHECK-AVX512-O3-NEXT: movl $2, %ecx
; CHECK-AVX512-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX512-O3-NEXT: vmovups (%rsp), %zmm0
; CHECK-AVX512-O3-NEXT: addq $72, %rsp
; CHECK-AVX512-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec8_double:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $72, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movl $64, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $2, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT: movapd (%rsp), %xmm0
; CHECK-SSE-O0-NEXT: movapd {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT: movapd {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O0-NEXT: movapd {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O0-NEXT: addq $72, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: atomic_vec8_double:
; CHECK-AVX2-O0: # %bb.0:
; CHECK-AVX2-O0-NEXT: subq $72, %rsp
; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT: movl $64, %edi
; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT: movl $2, %ecx
; CHECK-AVX2-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX2-O0-NEXT: vmovupd (%rsp), %ymm0
; CHECK-AVX2-O0-NEXT: vmovupd {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O0-NEXT: addq $72, %rsp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: atomic_vec8_double:
; CHECK-AVX512-O0: # %bb.0:
; CHECK-AVX512-O0-NEXT: subq $72, %rsp
; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT: movl $64, %edi
; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT: movl $2, %ecx
; CHECK-AVX512-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX512-O0-NEXT: vmovupd (%rsp), %zmm0
; CHECK-AVX512-O0-NEXT: addq $72, %rsp
; CHECK-AVX512-O0-NEXT: retq
%ret = load atomic <8 x double>, ptr %x acquire, align 4
ret <8 x double> %ret
}
define <16 x bfloat> @atomic_vec16_bfloat(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec16_bfloat:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $40, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $32, %edi
; CHECK-SSE-O3-NEXT: movl $2, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT: addq $40, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec16_bfloat:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: subq $40, %rsp
; CHECK-AVX-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX-O3-NEXT: movl $32, %edi
; CHECK-AVX-O3-NEXT: movl $2, %ecx
; CHECK-AVX-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT: vmovups (%rsp), %ymm0
; CHECK-AVX-O3-NEXT: addq $40, %rsp
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec16_bfloat:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $40, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movl $32, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $2, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT: addq $40, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec16_bfloat:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: subq $40, %rsp
; CHECK-AVX-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX-O0-NEXT: movl $32, %edi
; CHECK-AVX-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX-O0-NEXT: movl $2, %ecx
; CHECK-AVX-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT: vmovups (%rsp), %ymm0
; CHECK-AVX-O0-NEXT: addq $40, %rsp
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <16 x bfloat>, ptr %x acquire, align 4
ret <16 x bfloat> %ret
}
define <32 x half> @atomic_vec32_half(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec32_half:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $72, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $64, %edi
; CHECK-SSE-O3-NEXT: movl $2, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O3-NEXT: addq $72, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX2-O3-LABEL: atomic_vec32_half:
; CHECK-AVX2-O3: # %bb.0:
; CHECK-AVX2-O3-NEXT: subq $72, %rsp
; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT: movl $64, %edi
; CHECK-AVX2-O3-NEXT: movl $2, %ecx
; CHECK-AVX2-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX2-O3-NEXT: vmovups (%rsp), %ymm0
; CHECK-AVX2-O3-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O3-NEXT: addq $72, %rsp
; CHECK-AVX2-O3-NEXT: retq
;
; CHECK-AVX512-O3-LABEL: atomic_vec32_half:
; CHECK-AVX512-O3: # %bb.0:
; CHECK-AVX512-O3-NEXT: subq $72, %rsp
; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT: movl $64, %edi
; CHECK-AVX512-O3-NEXT: movl $2, %ecx
; CHECK-AVX512-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX512-O3-NEXT: vmovups (%rsp), %zmm0
; CHECK-AVX512-O3-NEXT: addq $72, %rsp
; CHECK-AVX512-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec32_half:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $72, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movl $64, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $2, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O0-NEXT: addq $72, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: atomic_vec32_half:
; CHECK-AVX2-O0: # %bb.0:
; CHECK-AVX2-O0-NEXT: subq $72, %rsp
; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT: movl $64, %edi
; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT: movl $2, %ecx
; CHECK-AVX2-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX2-O0-NEXT: vmovups (%rsp), %ymm0
; CHECK-AVX2-O0-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O0-NEXT: addq $72, %rsp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: atomic_vec32_half:
; CHECK-AVX512-O0: # %bb.0:
; CHECK-AVX512-O0-NEXT: subq $72, %rsp
; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT: movl $64, %edi
; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT: movl $2, %ecx
; CHECK-AVX512-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX512-O0-NEXT: vmovups (%rsp), %zmm0
; CHECK-AVX512-O0-NEXT: addq $72, %rsp
; CHECK-AVX512-O0-NEXT: retq
%ret = load atomic <32 x half>, ptr %x acquire, align 4
ret <32 x half> %ret
}
define <8 x float> @atomic_vec8_float_align(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec8_float_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $40, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $32, %edi
; CHECK-SSE-O3-NEXT: movl $2, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT: addq $40, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec8_float_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: subq $40, %rsp
; CHECK-AVX-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX-O3-NEXT: movl $32, %edi
; CHECK-AVX-O3-NEXT: movl $2, %ecx
; CHECK-AVX-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT: vmovups (%rsp), %ymm0
; CHECK-AVX-O3-NEXT: addq $40, %rsp
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec8_float_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $40, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movl $32, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $2, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT: addq $40, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec8_float_align:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: subq $40, %rsp
; CHECK-AVX-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX-O0-NEXT: movl $32, %edi
; CHECK-AVX-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX-O0-NEXT: movl $2, %ecx
; CHECK-AVX-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT: vmovups (%rsp), %ymm0
; CHECK-AVX-O0-NEXT: addq $40, %rsp
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <8 x float>, ptr %x acquire, align 32
ret <8 x float> %ret
}
define <16 x float> @atomic_vec16_float_align(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec16_float_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $72, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $64, %edi
; CHECK-SSE-O3-NEXT: movl $2, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O3-NEXT: addq $72, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX2-O3-LABEL: atomic_vec16_float_align:
; CHECK-AVX2-O3: # %bb.0:
; CHECK-AVX2-O3-NEXT: subq $72, %rsp
; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT: movl $64, %edi
; CHECK-AVX2-O3-NEXT: movl $2, %ecx
; CHECK-AVX2-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX2-O3-NEXT: vmovups (%rsp), %ymm0
; CHECK-AVX2-O3-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O3-NEXT: addq $72, %rsp
; CHECK-AVX2-O3-NEXT: retq
;
; CHECK-AVX512-O3-LABEL: atomic_vec16_float_align:
; CHECK-AVX512-O3: # %bb.0:
; CHECK-AVX512-O3-NEXT: subq $72, %rsp
; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT: movl $64, %edi
; CHECK-AVX512-O3-NEXT: movl $2, %ecx
; CHECK-AVX512-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX512-O3-NEXT: vmovups (%rsp), %zmm0
; CHECK-AVX512-O3-NEXT: addq $72, %rsp
; CHECK-AVX512-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec16_float_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $72, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movl $64, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $2, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O0-NEXT: addq $72, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: atomic_vec16_float_align:
; CHECK-AVX2-O0: # %bb.0:
; CHECK-AVX2-O0-NEXT: subq $72, %rsp
; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT: movl $64, %edi
; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT: movl $2, %ecx
; CHECK-AVX2-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX2-O0-NEXT: vmovups (%rsp), %ymm0
; CHECK-AVX2-O0-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O0-NEXT: addq $72, %rsp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: atomic_vec16_float_align:
; CHECK-AVX512-O0: # %bb.0:
; CHECK-AVX512-O0-NEXT: subq $72, %rsp
; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT: movl $64, %edi
; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT: movl $2, %ecx
; CHECK-AVX512-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX512-O0-NEXT: vmovups (%rsp), %zmm0
; CHECK-AVX512-O0-NEXT: addq $72, %rsp
; CHECK-AVX512-O0-NEXT: retq
%ret = load atomic <16 x float>, ptr %x acquire, align 64
ret <16 x float> %ret
}
define <8 x double> @atomic_vec8_double_align(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec8_double_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $72, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $64, %edi
; CHECK-SSE-O3-NEXT: movl $2, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O3-NEXT: addq $72, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX2-O3-LABEL: atomic_vec8_double_align:
; CHECK-AVX2-O3: # %bb.0:
; CHECK-AVX2-O3-NEXT: subq $72, %rsp
; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT: movl $64, %edi
; CHECK-AVX2-O3-NEXT: movl $2, %ecx
; CHECK-AVX2-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX2-O3-NEXT: vmovups (%rsp), %ymm0
; CHECK-AVX2-O3-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O3-NEXT: addq $72, %rsp
; CHECK-AVX2-O3-NEXT: retq
;
; CHECK-AVX512-O3-LABEL: atomic_vec8_double_align:
; CHECK-AVX512-O3: # %bb.0:
; CHECK-AVX512-O3-NEXT: subq $72, %rsp
; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT: movl $64, %edi
; CHECK-AVX512-O3-NEXT: movl $2, %ecx
; CHECK-AVX512-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX512-O3-NEXT: vmovups (%rsp), %zmm0
; CHECK-AVX512-O3-NEXT: addq $72, %rsp
; CHECK-AVX512-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec8_double_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $72, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movl $64, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $2, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT: movapd (%rsp), %xmm0
; CHECK-SSE-O0-NEXT: movapd {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT: movapd {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O0-NEXT: movapd {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O0-NEXT: addq $72, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: atomic_vec8_double_align:
; CHECK-AVX2-O0: # %bb.0:
; CHECK-AVX2-O0-NEXT: subq $72, %rsp
; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT: movl $64, %edi
; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT: movl $2, %ecx
; CHECK-AVX2-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX2-O0-NEXT: vmovupd (%rsp), %ymm0
; CHECK-AVX2-O0-NEXT: vmovupd {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O0-NEXT: addq $72, %rsp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: atomic_vec8_double_align:
; CHECK-AVX512-O0: # %bb.0:
; CHECK-AVX512-O0-NEXT: subq $72, %rsp
; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT: movl $64, %edi
; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT: movl $2, %ecx
; CHECK-AVX512-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX512-O0-NEXT: vmovupd (%rsp), %zmm0
; CHECK-AVX512-O0-NEXT: addq $72, %rsp
; CHECK-AVX512-O0-NEXT: retq
%ret = load atomic <8 x double>, ptr %x acquire, align 64
ret <8 x double> %ret
}
define <16 x bfloat> @atomic_vec16_bfloat_align(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec16_bfloat_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $40, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $32, %edi
; CHECK-SSE-O3-NEXT: movl $2, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT: addq $40, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec16_bfloat_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: subq $40, %rsp
; CHECK-AVX-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX-O3-NEXT: movl $32, %edi
; CHECK-AVX-O3-NEXT: movl $2, %ecx
; CHECK-AVX-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT: vmovups (%rsp), %ymm0
; CHECK-AVX-O3-NEXT: addq $40, %rsp
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec16_bfloat_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $40, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movl $32, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $2, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT: addq $40, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec16_bfloat_align:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: subq $40, %rsp
; CHECK-AVX-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX-O0-NEXT: movl $32, %edi
; CHECK-AVX-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX-O0-NEXT: movl $2, %ecx
; CHECK-AVX-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT: vmovups (%rsp), %ymm0
; CHECK-AVX-O0-NEXT: addq $40, %rsp
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <16 x bfloat>, ptr %x acquire, align 32
ret <16 x bfloat> %ret
}
define <32 x half> @atomic_vec32_half_align(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec32_half_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $72, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $64, %edi
; CHECK-SSE-O3-NEXT: movl $2, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O3-NEXT: addq $72, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX2-O3-LABEL: atomic_vec32_half_align:
; CHECK-AVX2-O3: # %bb.0:
; CHECK-AVX2-O3-NEXT: subq $72, %rsp
; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT: movl $64, %edi
; CHECK-AVX2-O3-NEXT: movl $2, %ecx
; CHECK-AVX2-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX2-O3-NEXT: vmovups (%rsp), %ymm0
; CHECK-AVX2-O3-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O3-NEXT: addq $72, %rsp
; CHECK-AVX2-O3-NEXT: retq
;
; CHECK-AVX512-O3-LABEL: atomic_vec32_half_align:
; CHECK-AVX512-O3: # %bb.0:
; CHECK-AVX512-O3-NEXT: subq $72, %rsp
; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT: movl $64, %edi
; CHECK-AVX512-O3-NEXT: movl $2, %ecx
; CHECK-AVX512-O3-NEXT: callq __atomic_load@PLT
; CHECK-AVX512-O3-NEXT: vmovups (%rsp), %zmm0
; CHECK-AVX512-O3-NEXT: addq $72, %rsp
; CHECK-AVX512-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec32_half_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $72, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movl $64, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $2, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT: movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O0-NEXT: addq $72, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: atomic_vec32_half_align:
; CHECK-AVX2-O0: # %bb.0:
; CHECK-AVX2-O0-NEXT: subq $72, %rsp
; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT: movl $64, %edi
; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT: movl $2, %ecx
; CHECK-AVX2-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX2-O0-NEXT: vmovups (%rsp), %ymm0
; CHECK-AVX2-O0-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O0-NEXT: addq $72, %rsp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: atomic_vec32_half_align:
; CHECK-AVX512-O0: # %bb.0:
; CHECK-AVX512-O0-NEXT: subq $72, %rsp
; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT: movl $64, %edi
; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT: movl $2, %ecx
; CHECK-AVX512-O0-NEXT: callq __atomic_load@PLT
; CHECK-AVX512-O0-NEXT: vmovups (%rsp), %zmm0
; CHECK-AVX512-O0-NEXT: addq $72, %rsp
; CHECK-AVX512-O0-NEXT: retq
%ret = load atomic <32 x half>, ptr %x acquire, align 64
ret <32 x half> %ret
}
define void @store_atomic_vec1_ptr_unaligned(ptr %x, <1 x ptr> %v) nounwind {
; CHECK-O3-LABEL: store_atomic_vec1_ptr_unaligned:
; CHECK-O3: # %bb.0:
; CHECK-O3-NEXT: pushq %rax
; CHECK-O3-NEXT: movq %rdi, %rax
; CHECK-O3-NEXT: movq %rsi, (%rsp)
; CHECK-O3-NEXT: movq %rsp, %rdx
; CHECK-O3-NEXT: movl $8, %edi
; CHECK-O3-NEXT: movq %rax, %rsi
; CHECK-O3-NEXT: movl $3, %ecx
; CHECK-O3-NEXT: callq __atomic_store@PLT
; CHECK-O3-NEXT: popq %rax
; CHECK-O3-NEXT: retq
;
; CHECK-O0-LABEL: store_atomic_vec1_ptr_unaligned:
; CHECK-O0: # %bb.0:
; CHECK-O0-NEXT: pushq %rax
; CHECK-O0-NEXT: movq %rsi, %rax
; CHECK-O0-NEXT: movq %rdi, %rsi
; CHECK-O0-NEXT: movq %rax, (%rsp)
; CHECK-O0-NEXT: movl $8, %edi
; CHECK-O0-NEXT: movq %rsp, %rdx
; CHECK-O0-NEXT: movl $3, %ecx
; CHECK-O0-NEXT: callq __atomic_store@PLT
; CHECK-O0-NEXT: popq %rax
; CHECK-O0-NEXT: retq
store atomic <1 x ptr> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec1_i64_unaligned(ptr %x, <1 x i64> %v) nounwind {
; CHECK-O3-LABEL: store_atomic_vec1_i64_unaligned:
; CHECK-O3: # %bb.0:
; CHECK-O3-NEXT: pushq %rax
; CHECK-O3-NEXT: movq %rdi, %rax
; CHECK-O3-NEXT: movq %rsi, (%rsp)
; CHECK-O3-NEXT: movq %rsp, %rdx
; CHECK-O3-NEXT: movl $8, %edi
; CHECK-O3-NEXT: movq %rax, %rsi
; CHECK-O3-NEXT: movl $3, %ecx
; CHECK-O3-NEXT: callq __atomic_store@PLT
; CHECK-O3-NEXT: popq %rax
; CHECK-O3-NEXT: retq
;
; CHECK-O0-LABEL: store_atomic_vec1_i64_unaligned:
; CHECK-O0: # %bb.0:
; CHECK-O0-NEXT: pushq %rax
; CHECK-O0-NEXT: movq %rsi, %rax
; CHECK-O0-NEXT: movq %rdi, %rsi
; CHECK-O0-NEXT: movq %rax, (%rsp)
; CHECK-O0-NEXT: movl $8, %edi
; CHECK-O0-NEXT: movq %rsp, %rdx
; CHECK-O0-NEXT: movl $3, %ecx
; CHECK-O0-NEXT: callq __atomic_store@PLT
; CHECK-O0-NEXT: popq %rax
; CHECK-O0-NEXT: retq
store atomic <1 x i64> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec1_double_unaligned(ptr %x, <1 x double> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_double_unaligned:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: pushq %rax
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movsd %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $8, %edi
; CHECK-SSE-O3-NEXT: movl $3, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT: popq %rax
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_double_unaligned:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: pushq %rax
; CHECK-AVX-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX-O3-NEXT: vmovsd %xmm0, (%rsp)
; CHECK-AVX-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX-O3-NEXT: movl $8, %edi
; CHECK-AVX-O3-NEXT: movl $3, %ecx
; CHECK-AVX-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT: popq %rax
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_double_unaligned:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: pushq %rax
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movsd %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT: movl $8, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $3, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT: popq %rax
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_double_unaligned:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: pushq %rax
; CHECK-AVX-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX-O0-NEXT: vmovsd %xmm0, (%rsp)
; CHECK-AVX-O0-NEXT: movl $8, %edi
; CHECK-AVX-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX-O0-NEXT: movl $3, %ecx
; CHECK-AVX-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX-O0-NEXT: popq %rax
; CHECK-AVX-O0-NEXT: retq
store atomic <1 x double> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec2_i32_unaligned(ptr %x, <2 x i32> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_i32_unaligned:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: pushq %rax
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $8, %edi
; CHECK-SSE-O3-NEXT: movl $3, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT: popq %rax
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i32_unaligned:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: pushq %rax
; CHECK-AVX-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rsp)
; CHECK-AVX-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX-O3-NEXT: movl $8, %edi
; CHECK-AVX-O3-NEXT: movl $3, %ecx
; CHECK-AVX-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT: popq %rax
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_i32_unaligned:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: pushq %rax
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movq %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT: movl $8, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $3, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT: popq %rax
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i32_unaligned:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: pushq %rax
; CHECK-AVX-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rsp)
; CHECK-AVX-O0-NEXT: movl $8, %edi
; CHECK-AVX-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX-O0-NEXT: movl $3, %ecx
; CHECK-AVX-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX-O0-NEXT: popq %rax
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x i32> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec4_float_unaligned(ptr %x, <4 x float> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_float_unaligned:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $24, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $16, %edi
; CHECK-SSE-O3-NEXT: movl $3, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT: addq $24, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_float_unaligned:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: subq $24, %rsp
; CHECK-AVX-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX-O3-NEXT: vmovaps %xmm0, (%rsp)
; CHECK-AVX-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX-O3-NEXT: movl $16, %edi
; CHECK-AVX-O3-NEXT: movl $3, %ecx
; CHECK-AVX-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT: addq $24, %rsp
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec4_float_unaligned:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $24, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movaps %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT: movl $16, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $3, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT: addq $24, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_float_unaligned:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: subq $24, %rsp
; CHECK-AVX-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX-O0-NEXT: vmovaps %xmm0, (%rsp)
; CHECK-AVX-O0-NEXT: movl $16, %edi
; CHECK-AVX-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX-O0-NEXT: movl $3, %ecx
; CHECK-AVX-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX-O0-NEXT: addq $24, %rsp
; CHECK-AVX-O0-NEXT: retq
store atomic <4 x float> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec8_double_unaligned(ptr %x, <8 x double> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $72, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $64, %edi
; CHECK-SSE-O3-NEXT: movl $3, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT: addq $72, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX2-O3-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-AVX2-O3: # %bb.0:
; CHECK-AVX2-O3-NEXT: subq $72, %rsp
; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O3-NEXT: vmovups %ymm0, (%rsp)
; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT: movl $64, %edi
; CHECK-AVX2-O3-NEXT: movl $3, %ecx
; CHECK-AVX2-O3-NEXT: vzeroupper
; CHECK-AVX2-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX2-O3-NEXT: addq $72, %rsp
; CHECK-AVX2-O3-NEXT: retq
;
; CHECK-AVX512-O3-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-AVX512-O3: # %bb.0:
; CHECK-AVX512-O3-NEXT: subq $72, %rsp
; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT: vmovups %zmm0, (%rsp)
; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT: movl $64, %edi
; CHECK-AVX512-O3-NEXT: movl $3, %ecx
; CHECK-AVX512-O3-NEXT: vzeroupper
; CHECK-AVX512-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX512-O3-NEXT: addq $72, %rsp
; CHECK-AVX512-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $72, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movapd %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT: movapd %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movapd %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movapd %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movl $64, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $3, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT: addq $72, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-AVX2-O0: # %bb.0:
; CHECK-AVX2-O0-NEXT: subq $72, %rsp
; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT: vmovupd %ymm0, (%rsp)
; CHECK-AVX2-O0-NEXT: vmovupd %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT: movl $64, %edi
; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT: movl $3, %ecx
; CHECK-AVX2-O0-NEXT: vzeroupper
; CHECK-AVX2-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT: addq $72, %rsp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-AVX512-O0: # %bb.0:
; CHECK-AVX512-O0-NEXT: subq $72, %rsp
; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT: vmovupd %zmm0, (%rsp)
; CHECK-AVX512-O0-NEXT: movl $64, %edi
; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT: movl $3, %ecx
; CHECK-AVX512-O0-NEXT: vzeroupper
; CHECK-AVX512-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT: addq $72, %rsp
; CHECK-AVX512-O0-NEXT: retq
store atomic <8 x double> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec16_bfloat_unaligned(ptr %x, <16 x bfloat> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $40, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $32, %edi
; CHECK-SSE-O3-NEXT: movl $3, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT: addq $40, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: subq $40, %rsp
; CHECK-AVX-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX-O3-NEXT: vmovups %ymm0, (%rsp)
; CHECK-AVX-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX-O3-NEXT: movl $32, %edi
; CHECK-AVX-O3-NEXT: movl $3, %ecx
; CHECK-AVX-O3-NEXT: vzeroupper
; CHECK-AVX-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT: addq $40, %rsp
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: subq $40, %rsp
; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm6
; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm14
; CHECK-SSE2-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE2-O0-NEXT: xorps %xmm11, %xmm11
; CHECK-SSE2-O0-NEXT: movaps %xmm11, %xmm3
; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm14[4],xmm11[5],xmm14[5],xmm11[6],xmm14[6],xmm11[7],xmm14[7]
; CHECK-SSE2-O0-NEXT: movaps %xmm6, %xmm7
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm7
; CHECK-SSE2-O0-NEXT: movaps %xmm14, %xmm15
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm15
; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm6, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm6, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm6, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm6, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm4
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm4
; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm6, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm5
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm5
; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm6, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm6
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm6
; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm14, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm8
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm8
; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm14, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm9
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm9
; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm14, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm10
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm10
; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm14, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm12
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm12
; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm14, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm13
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm13
; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm14, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm14
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm14
; CHECK-SSE2-O0-NEXT: movd %xmm15, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, (%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm14, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm13, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm12, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm11, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm10, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm9, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm8, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm7, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm6, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm5, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm4, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm3, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm2, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm1, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movl $32, %edi
; CHECK-SSE2-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE2-O0-NEXT: movl $3, %ecx
; CHECK-SSE2-O0-NEXT: callq __atomic_store@PLT
; CHECK-SSE2-O0-NEXT: addq $40, %rsp
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: subq $40, %rsp
; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm5
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm13
; CHECK-SSE4-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm3
; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
; CHECK-SSE4-O0-NEXT: movaps %xmm5, %xmm6
; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm6 = xmm0[0],xmm6[1],xmm0[2],xmm6[3],xmm0[4],xmm6[5],xmm0[6],xmm6[7]
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm11
; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm13[4],xmm11[5],xmm13[5],xmm11[6],xmm13[6],xmm11[7],xmm13[7]
; CHECK-SSE4-O0-NEXT: movaps %xmm13, %xmm14
; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm14 = xmm0[0],xmm14[1],xmm0[2],xmm14[3],xmm0[4],xmm14[5],xmm0[6],xmm14[7]
; CHECK-SSE4-O0-NEXT: movaps %xmm5, %xmm7
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm7
; CHECK-SSE4-O0-NEXT: movaps %xmm13, %xmm15
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm15
; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm5, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm5, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm5, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2
; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm5, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm4
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm4
; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm5, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm5
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm5
; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm13, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm8
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm8
; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm13, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm9
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm9
; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm13, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm10
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm10
; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm13, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm12
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm12
; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm13, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm13
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm13
; CHECK-SSE4-O0-NEXT: movd %xmm15, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, (%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm14, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm13, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm12, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm11, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm10, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm9, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm8, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm7, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm6, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm5, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm4, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm3, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm2, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm1, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm0, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movl $32, %edi
; CHECK-SSE4-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE4-O0-NEXT: movl $3, %ecx
; CHECK-SSE4-O0-NEXT: callq __atomic_store@PLT
; CHECK-SSE4-O0-NEXT: addq $40, %rsp
; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-AVX2-O0: # %bb.0:
; CHECK-AVX2-O0-NEXT: subq $40, %rsp
; CHECK-AVX2-O0-NEXT: vmovaps %ymm0, %ymm1
; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm1, %xmm0
; CHECK-AVX2-O0-NEXT: # kill: def $xmm1 killed $xmm1 killed $ymm1
; CHECK-AVX2-O0-NEXT: vmovaps %xmm1, (%rsp)
; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT: movl $32, %edi
; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT: movl $3, %ecx
; CHECK-AVX2-O0-NEXT: vzeroupper
; CHECK-AVX2-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT: addq $40, %rsp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-AVX512-O0: # %bb.0:
; CHECK-AVX512-O0-NEXT: subq $40, %rsp
; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT: vmovups %ymm0, (%rsp)
; CHECK-AVX512-O0-NEXT: movl $32, %edi
; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT: movl $3, %ecx
; CHECK-AVX512-O0-NEXT: vzeroupper
; CHECK-AVX512-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT: addq $40, %rsp
; CHECK-AVX512-O0-NEXT: retq
store atomic <16 x bfloat> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec32_half_unaligned(ptr %x, <32 x half> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $72, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $64, %edi
; CHECK-SSE-O3-NEXT: movl $3, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT: addq $72, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX2-O3-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-AVX2-O3: # %bb.0:
; CHECK-AVX2-O3-NEXT: subq $72, %rsp
; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O3-NEXT: vmovups %ymm0, (%rsp)
; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT: movl $64, %edi
; CHECK-AVX2-O3-NEXT: movl $3, %ecx
; CHECK-AVX2-O3-NEXT: vzeroupper
; CHECK-AVX2-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX2-O3-NEXT: addq $72, %rsp
; CHECK-AVX2-O3-NEXT: retq
;
; CHECK-AVX512-O3-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-AVX512-O3: # %bb.0:
; CHECK-AVX512-O3-NEXT: subq $72, %rsp
; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT: vmovups %zmm0, (%rsp)
; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT: movl $64, %edi
; CHECK-AVX512-O3-NEXT: movl $3, %ecx
; CHECK-AVX512-O3-NEXT: vzeroupper
; CHECK-AVX512-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX512-O3-NEXT: addq $72, %rsp
; CHECK-AVX512-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $72, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movaps %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movl $64, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $3, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT: addq $72, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-AVX2-O0: # %bb.0:
; CHECK-AVX2-O0-NEXT: subq $72, %rsp
; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT: vmovups %ymm0, (%rsp)
; CHECK-AVX2-O0-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT: movl $64, %edi
; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT: movl $3, %ecx
; CHECK-AVX2-O0-NEXT: vzeroupper
; CHECK-AVX2-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT: addq $72, %rsp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-AVX512-O0: # %bb.0:
; CHECK-AVX512-O0-NEXT: subq $72, %rsp
; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT: vmovups %zmm0, (%rsp)
; CHECK-AVX512-O0-NEXT: movl $64, %edi
; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT: movl $3, %ecx
; CHECK-AVX512-O0-NEXT: vzeroupper
; CHECK-AVX512-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT: addq $72, %rsp
; CHECK-AVX512-O0-NEXT: retq
store atomic <32 x half> %v, ptr %x release, align 4
ret void
}
define void @store_atomic_vec8_float_align(ptr %x, <8 x float> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec8_float_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $40, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $32, %edi
; CHECK-SSE-O3-NEXT: movl $3, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT: addq $40, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec8_float_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: subq $40, %rsp
; CHECK-AVX-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX-O3-NEXT: vmovups %ymm0, (%rsp)
; CHECK-AVX-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX-O3-NEXT: movl $32, %edi
; CHECK-AVX-O3-NEXT: movl $3, %ecx
; CHECK-AVX-O3-NEXT: vzeroupper
; CHECK-AVX-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT: addq $40, %rsp
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec8_float_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $40, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movaps %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movl $32, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $3, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT: addq $40, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec8_float_align:
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: subq $40, %rsp
; CHECK-AVX-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX-O0-NEXT: vmovups %ymm0, (%rsp)
; CHECK-AVX-O0-NEXT: movl $32, %edi
; CHECK-AVX-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX-O0-NEXT: movl $3, %ecx
; CHECK-AVX-O0-NEXT: vzeroupper
; CHECK-AVX-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX-O0-NEXT: addq $40, %rsp
; CHECK-AVX-O0-NEXT: retq
store atomic <8 x float> %v, ptr %x release, align 32
ret void
}
define void @store_atomic_vec16_float_align(ptr %x, <16 x float> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec16_float_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $72, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $64, %edi
; CHECK-SSE-O3-NEXT: movl $3, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT: addq $72, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX2-O3-LABEL: store_atomic_vec16_float_align:
; CHECK-AVX2-O3: # %bb.0:
; CHECK-AVX2-O3-NEXT: subq $72, %rsp
; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O3-NEXT: vmovups %ymm0, (%rsp)
; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT: movl $64, %edi
; CHECK-AVX2-O3-NEXT: movl $3, %ecx
; CHECK-AVX2-O3-NEXT: vzeroupper
; CHECK-AVX2-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX2-O3-NEXT: addq $72, %rsp
; CHECK-AVX2-O3-NEXT: retq
;
; CHECK-AVX512-O3-LABEL: store_atomic_vec16_float_align:
; CHECK-AVX512-O3: # %bb.0:
; CHECK-AVX512-O3-NEXT: subq $72, %rsp
; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT: vmovups %zmm0, (%rsp)
; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT: movl $64, %edi
; CHECK-AVX512-O3-NEXT: movl $3, %ecx
; CHECK-AVX512-O3-NEXT: vzeroupper
; CHECK-AVX512-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX512-O3-NEXT: addq $72, %rsp
; CHECK-AVX512-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec16_float_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $72, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movaps %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movl $64, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $3, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT: addq $72, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec16_float_align:
; CHECK-AVX2-O0: # %bb.0:
; CHECK-AVX2-O0-NEXT: subq $72, %rsp
; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT: vmovups %ymm0, (%rsp)
; CHECK-AVX2-O0-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT: movl $64, %edi
; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT: movl $3, %ecx
; CHECK-AVX2-O0-NEXT: vzeroupper
; CHECK-AVX2-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT: addq $72, %rsp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec16_float_align:
; CHECK-AVX512-O0: # %bb.0:
; CHECK-AVX512-O0-NEXT: subq $72, %rsp
; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT: vmovups %zmm0, (%rsp)
; CHECK-AVX512-O0-NEXT: movl $64, %edi
; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT: movl $3, %ecx
; CHECK-AVX512-O0-NEXT: vzeroupper
; CHECK-AVX512-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT: addq $72, %rsp
; CHECK-AVX512-O0-NEXT: retq
store atomic <16 x float> %v, ptr %x release, align 64
ret void
}
define void @store_atomic_vec8_double_align(ptr %x, <8 x double> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec8_double_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $72, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $64, %edi
; CHECK-SSE-O3-NEXT: movl $3, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT: addq $72, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX2-O3-LABEL: store_atomic_vec8_double_align:
; CHECK-AVX2-O3: # %bb.0:
; CHECK-AVX2-O3-NEXT: subq $72, %rsp
; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O3-NEXT: vmovups %ymm0, (%rsp)
; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT: movl $64, %edi
; CHECK-AVX2-O3-NEXT: movl $3, %ecx
; CHECK-AVX2-O3-NEXT: vzeroupper
; CHECK-AVX2-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX2-O3-NEXT: addq $72, %rsp
; CHECK-AVX2-O3-NEXT: retq
;
; CHECK-AVX512-O3-LABEL: store_atomic_vec8_double_align:
; CHECK-AVX512-O3: # %bb.0:
; CHECK-AVX512-O3-NEXT: subq $72, %rsp
; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT: vmovups %zmm0, (%rsp)
; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT: movl $64, %edi
; CHECK-AVX512-O3-NEXT: movl $3, %ecx
; CHECK-AVX512-O3-NEXT: vzeroupper
; CHECK-AVX512-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX512-O3-NEXT: addq $72, %rsp
; CHECK-AVX512-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec8_double_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $72, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movapd %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT: movapd %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movapd %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movapd %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movl $64, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $3, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT: addq $72, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec8_double_align:
; CHECK-AVX2-O0: # %bb.0:
; CHECK-AVX2-O0-NEXT: subq $72, %rsp
; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT: vmovupd %ymm0, (%rsp)
; CHECK-AVX2-O0-NEXT: vmovupd %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT: movl $64, %edi
; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT: movl $3, %ecx
; CHECK-AVX2-O0-NEXT: vzeroupper
; CHECK-AVX2-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT: addq $72, %rsp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec8_double_align:
; CHECK-AVX512-O0: # %bb.0:
; CHECK-AVX512-O0-NEXT: subq $72, %rsp
; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT: vmovupd %zmm0, (%rsp)
; CHECK-AVX512-O0-NEXT: movl $64, %edi
; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT: movl $3, %ecx
; CHECK-AVX512-O0-NEXT: vzeroupper
; CHECK-AVX512-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT: addq $72, %rsp
; CHECK-AVX512-O0-NEXT: retq
store atomic <8 x double> %v, ptr %x release, align 64
ret void
}
define void @store_atomic_vec16_bfloat_align(ptr %x, <16 x bfloat> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $40, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $32, %edi
; CHECK-SSE-O3-NEXT: movl $3, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT: addq $40, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: subq $40, %rsp
; CHECK-AVX-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX-O3-NEXT: vmovups %ymm0, (%rsp)
; CHECK-AVX-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX-O3-NEXT: movl $32, %edi
; CHECK-AVX-O3-NEXT: movl $3, %ecx
; CHECK-AVX-O3-NEXT: vzeroupper
; CHECK-AVX-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT: addq $40, %rsp
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: subq $40, %rsp
; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm6
; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm14
; CHECK-SSE2-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE2-O0-NEXT: xorps %xmm11, %xmm11
; CHECK-SSE2-O0-NEXT: movaps %xmm11, %xmm3
; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm14[4],xmm11[5],xmm14[5],xmm11[6],xmm14[6],xmm11[7],xmm14[7]
; CHECK-SSE2-O0-NEXT: movaps %xmm6, %xmm7
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm7
; CHECK-SSE2-O0-NEXT: movaps %xmm14, %xmm15
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm15
; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm6, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm6, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm6, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm6, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm4
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm4
; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm6, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm5
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm5
; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm6, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm6
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm6
; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm14, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm8
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm8
; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm14, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm9
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm9
; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm14, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm10
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm10
; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm14, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm12
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm12
; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm14, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm13
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm13
; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm14, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm14
; CHECK-SSE2-O0-NEXT: pslld $16, %xmm14
; CHECK-SSE2-O0-NEXT: movd %xmm15, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, (%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm14, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm13, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm12, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm11, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm10, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm9, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm8, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm7, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm6, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm5, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm4, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm3, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm2, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm1, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax
; CHECK-SSE2-O0-NEXT: shrl $16, %eax
; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT: movl $32, %edi
; CHECK-SSE2-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE2-O0-NEXT: movl $3, %ecx
; CHECK-SSE2-O0-NEXT: callq __atomic_store@PLT
; CHECK-SSE2-O0-NEXT: addq $40, %rsp
; CHECK-SSE2-O0-NEXT: retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: subq $40, %rsp
; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm5
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm13
; CHECK-SSE4-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm3
; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
; CHECK-SSE4-O0-NEXT: movaps %xmm5, %xmm6
; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm6 = xmm0[0],xmm6[1],xmm0[2],xmm6[3],xmm0[4],xmm6[5],xmm0[6],xmm6[7]
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm11
; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm13[4],xmm11[5],xmm13[5],xmm11[6],xmm13[6],xmm11[7],xmm13[7]
; CHECK-SSE4-O0-NEXT: movaps %xmm13, %xmm14
; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm14 = xmm0[0],xmm14[1],xmm0[2],xmm14[3],xmm0[4],xmm14[5],xmm0[6],xmm14[7]
; CHECK-SSE4-O0-NEXT: movaps %xmm5, %xmm7
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm7
; CHECK-SSE4-O0-NEXT: movaps %xmm13, %xmm15
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm15
; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm5, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm5, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm5, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2
; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm5, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm4
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm4
; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm5, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm5
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm5
; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm13, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm8
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm8
; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm13, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm9
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm9
; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm13, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm10
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm10
; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm13, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm12
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm12
; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm13, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm13
; CHECK-SSE4-O0-NEXT: pslld $16, %xmm13
; CHECK-SSE4-O0-NEXT: movd %xmm15, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, (%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm14, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm13, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm12, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm11, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm10, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm9, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm8, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm7, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm6, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm5, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm4, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm3, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm2, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm1, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movd %xmm0, %eax
; CHECK-SSE4-O0-NEXT: shrl $16, %eax
; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT: movl $32, %edi
; CHECK-SSE4-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE4-O0-NEXT: movl $3, %ecx
; CHECK-SSE4-O0-NEXT: callq __atomic_store@PLT
; CHECK-SSE4-O0-NEXT: addq $40, %rsp
; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-AVX2-O0: # %bb.0:
; CHECK-AVX2-O0-NEXT: subq $40, %rsp
; CHECK-AVX2-O0-NEXT: vmovaps %ymm0, %ymm1
; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm1, %xmm0
; CHECK-AVX2-O0-NEXT: # kill: def $xmm1 killed $xmm1 killed $ymm1
; CHECK-AVX2-O0-NEXT: vmovaps %xmm1, (%rsp)
; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT: movl $32, %edi
; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT: movl $3, %ecx
; CHECK-AVX2-O0-NEXT: vzeroupper
; CHECK-AVX2-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT: addq $40, %rsp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-AVX512-O0: # %bb.0:
; CHECK-AVX512-O0-NEXT: subq $40, %rsp
; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT: vmovups %ymm0, (%rsp)
; CHECK-AVX512-O0-NEXT: movl $32, %edi
; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT: movl $3, %ecx
; CHECK-AVX512-O0-NEXT: vzeroupper
; CHECK-AVX512-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT: addq $40, %rsp
; CHECK-AVX512-O0-NEXT: retq
store atomic <16 x bfloat> %v, ptr %x release, align 32
ret void
}
define void @store_atomic_vec32_half_align(ptr %x, <32 x half> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec32_half_align:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: subq $72, %rsp
; CHECK-SSE-O3-NEXT: movq %rdi, %rsi
; CHECK-SSE-O3-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT: movq %rsp, %rdx
; CHECK-SSE-O3-NEXT: movl $64, %edi
; CHECK-SSE-O3-NEXT: movl $3, %ecx
; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT: addq $72, %rsp
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX2-O3-LABEL: store_atomic_vec32_half_align:
; CHECK-AVX2-O3: # %bb.0:
; CHECK-AVX2-O3-NEXT: subq $72, %rsp
; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O3-NEXT: vmovups %ymm0, (%rsp)
; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT: movl $64, %edi
; CHECK-AVX2-O3-NEXT: movl $3, %ecx
; CHECK-AVX2-O3-NEXT: vzeroupper
; CHECK-AVX2-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX2-O3-NEXT: addq $72, %rsp
; CHECK-AVX2-O3-NEXT: retq
;
; CHECK-AVX512-O3-LABEL: store_atomic_vec32_half_align:
; CHECK-AVX512-O3: # %bb.0:
; CHECK-AVX512-O3-NEXT: subq $72, %rsp
; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT: vmovups %zmm0, (%rsp)
; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT: movl $64, %edi
; CHECK-AVX512-O3-NEXT: movl $3, %ecx
; CHECK-AVX512-O3-NEXT: vzeroupper
; CHECK-AVX512-O3-NEXT: callq __atomic_store@PLT
; CHECK-AVX512-O3-NEXT: addq $72, %rsp
; CHECK-AVX512-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec32_half_align:
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: subq $72, %rsp
; CHECK-SSE-O0-NEXT: movq %rdi, %rsi
; CHECK-SSE-O0-NEXT: movaps %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT: movl $64, %edi
; CHECK-SSE-O0-NEXT: movq %rsp, %rdx
; CHECK-SSE-O0-NEXT: movl $3, %ecx
; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT: addq $72, %rsp
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec32_half_align:
; CHECK-AVX2-O0: # %bb.0:
; CHECK-AVX2-O0-NEXT: subq $72, %rsp
; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT: vmovups %ymm0, (%rsp)
; CHECK-AVX2-O0-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT: movl $64, %edi
; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT: movl $3, %ecx
; CHECK-AVX2-O0-NEXT: vzeroupper
; CHECK-AVX2-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT: addq $72, %rsp
; CHECK-AVX2-O0-NEXT: retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec32_half_align:
; CHECK-AVX512-O0: # %bb.0:
; CHECK-AVX512-O0-NEXT: subq $72, %rsp
; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT: vmovups %zmm0, (%rsp)
; CHECK-AVX512-O0-NEXT: movl $64, %edi
; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT: movl $3, %ecx
; CHECK-AVX512-O0-NEXT: vzeroupper
; CHECK-AVX512-O0-NEXT: callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT: addq $72, %rsp
; CHECK-AVX512-O0-NEXT: retq
store atomic <32 x half> %v, ptr %x release, align 64
ret void
}