| ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py | |
| ; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-SSE-O3,CHECK-SSE2-O3 | |
| ; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-SSE-O3,CHECK-SSE4-O3 | |
| ; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-AVX-O3,CHECK-AVX2-O3 | |
| ; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-AVX-O3,CHECK-AVX512-O3 | |
| ; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-SSE-O0,CHECK-SSE2-O0 | |
| ; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-SSE-O0,CHECK-SSE4-O0 | |
| ; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-AVX-O0,CHECK-AVX2-O0 | |
| ; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-AVX-O0,CHECK-AVX512-O0 | |
| define void @test1(ptr %ptr, i32 %val1) { | |
| ; CHECK-LABEL: test1: | |
| ; CHECK: # %bb.0: | |
| ; CHECK-NEXT: xchgl %esi, (%rdi) | |
| ; CHECK-NEXT: retq | |
| store atomic i32 %val1, ptr %ptr seq_cst, align 4 | |
| ret void | |
| } | |
| define void @test2(ptr %ptr, i32 %val1) { | |
| ; CHECK-LABEL: test2: | |
| ; CHECK: # %bb.0: | |
| ; CHECK-NEXT: movl %esi, (%rdi) | |
| ; CHECK-NEXT: retq | |
| store atomic i32 %val1, ptr %ptr release, align 4 | |
| ret void | |
| } | |
| define i32 @test3(ptr %ptr) { | |
| ; CHECK-LABEL: test3: | |
| ; CHECK: # %bb.0: | |
| ; CHECK-NEXT: movl (%rdi), %eax | |
| ; CHECK-NEXT: retq | |
| %val = load atomic i32, ptr %ptr seq_cst, align 4 | |
| ret i32 %val | |
| } | |
| define <1 x i32> @atomic_vec1_i32(ptr %x) { | |
| ; CHECK-LABEL: atomic_vec1_i32: | |
| ; CHECK: # %bb.0: | |
| ; CHECK-NEXT: movl (%rdi), %eax | |
| ; CHECK-NEXT: retq | |
| %ret = load atomic <1 x i32>, ptr %x acquire, align 4 | |
| ret <1 x i32> %ret | |
| } | |
| define <1 x i8> @atomic_vec1_i8(ptr %x) { | |
| ; CHECK-O3-LABEL: atomic_vec1_i8: | |
| ; CHECK-O3: # %bb.0: | |
| ; CHECK-O3-NEXT: movzbl (%rdi), %eax | |
| ; CHECK-O3-NEXT: retq | |
| ; | |
| ; CHECK-O0-LABEL: atomic_vec1_i8: | |
| ; CHECK-O0: # %bb.0: | |
| ; CHECK-O0-NEXT: movb (%rdi), %al | |
| ; CHECK-O0-NEXT: retq | |
| %ret = load atomic <1 x i8>, ptr %x acquire, align 1 | |
| ret <1 x i8> %ret | |
| } | |
| define <1 x i16> @atomic_vec1_i16(ptr %x) { | |
| ; CHECK-O3-LABEL: atomic_vec1_i16: | |
| ; CHECK-O3: # %bb.0: | |
| ; CHECK-O3-NEXT: movzwl (%rdi), %eax | |
| ; CHECK-O3-NEXT: retq | |
| ; | |
| ; CHECK-O0-LABEL: atomic_vec1_i16: | |
| ; CHECK-O0: # %bb.0: | |
| ; CHECK-O0-NEXT: movw (%rdi), %ax | |
| ; CHECK-O0-NEXT: retq | |
| %ret = load atomic <1 x i16>, ptr %x acquire, align 2 | |
| ret <1 x i16> %ret | |
| } | |
| define <1 x i32> @atomic_vec1_i8_zext(ptr %x) { | |
| ; CHECK-O3-LABEL: atomic_vec1_i8_zext: | |
| ; CHECK-O3: # %bb.0: | |
| ; CHECK-O3-NEXT: movzbl (%rdi), %eax | |
| ; CHECK-O3-NEXT: movzbl %al, %eax | |
| ; CHECK-O3-NEXT: retq | |
| ; | |
| ; CHECK-O0-LABEL: atomic_vec1_i8_zext: | |
| ; CHECK-O0: # %bb.0: | |
| ; CHECK-O0-NEXT: movb (%rdi), %al | |
| ; CHECK-O0-NEXT: movzbl %al, %eax | |
| ; CHECK-O0-NEXT: retq | |
| %ret = load atomic <1 x i8>, ptr %x acquire, align 1 | |
| %zret = zext <1 x i8> %ret to <1 x i32> | |
| ret <1 x i32> %zret | |
| } | |
| define <1 x i64> @atomic_vec1_i16_sext(ptr %x) { | |
| ; CHECK-O3-LABEL: atomic_vec1_i16_sext: | |
| ; CHECK-O3: # %bb.0: | |
| ; CHECK-O3-NEXT: movzwl (%rdi), %eax | |
| ; CHECK-O3-NEXT: movswq %ax, %rax | |
| ; CHECK-O3-NEXT: retq | |
| ; | |
| ; CHECK-O0-LABEL: atomic_vec1_i16_sext: | |
| ; CHECK-O0: # %bb.0: | |
| ; CHECK-O0-NEXT: movw (%rdi), %ax | |
| ; CHECK-O0-NEXT: movswq %ax, %rax | |
| ; CHECK-O0-NEXT: retq | |
| %ret = load atomic <1 x i16>, ptr %x acquire, align 2 | |
| %sret = sext <1 x i16> %ret to <1 x i64> | |
| ret <1 x i64> %sret | |
| } | |
| define <1 x ptr addrspace(270)> @atomic_vec1_ptr270(ptr %x) { | |
| ; CHECK-LABEL: atomic_vec1_ptr270: | |
| ; CHECK: # %bb.0: | |
| ; CHECK-NEXT: movl (%rdi), %eax | |
| ; CHECK-NEXT: retq | |
| %ret = load atomic <1 x ptr addrspace(270)>, ptr %x acquire, align 4 | |
| ret <1 x ptr addrspace(270)> %ret | |
| } | |
| define <1 x bfloat> @atomic_vec1_bfloat(ptr %x) { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec1_bfloat: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movzwl (%rdi), %eax | |
| ; CHECK-SSE-O3-NEXT: movd %eax, %xmm0 | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec1_bfloat: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: movzwl (%rdi), %eax | |
| ; CHECK-AVX-O3-NEXT: vmovd %eax, %xmm0 | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec1_bfloat: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movw (%rdi), %cx | |
| ; CHECK-SSE-O0-NEXT: # implicit-def: $eax | |
| ; CHECK-SSE-O0-NEXT: movw %cx, %ax | |
| ; CHECK-SSE-O0-NEXT: movd %eax, %xmm0 | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec1_bfloat: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: movw (%rdi), %cx | |
| ; CHECK-AVX-O0-NEXT: # implicit-def: $eax | |
| ; CHECK-AVX-O0-NEXT: movw %cx, %ax | |
| ; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0 | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <1 x bfloat>, ptr %x acquire, align 2 | |
| ret <1 x bfloat> %ret | |
| } | |
| define <1 x ptr> @atomic_vec1_ptr_align(ptr %x) nounwind { | |
| ; CHECK-LABEL: atomic_vec1_ptr_align: | |
| ; CHECK: # %bb.0: | |
| ; CHECK-NEXT: movq (%rdi), %rax | |
| ; CHECK-NEXT: retq | |
| %ret = load atomic <1 x ptr>, ptr %x acquire, align 8 | |
| ret <1 x ptr> %ret | |
| } | |
| define <1 x i64> @atomic_vec1_i64_align(ptr %x) nounwind { | |
| ; CHECK-LABEL: atomic_vec1_i64_align: | |
| ; CHECK: # %bb.0: | |
| ; CHECK-NEXT: movq (%rdi), %rax | |
| ; CHECK-NEXT: retq | |
| %ret = load atomic <1 x i64>, ptr %x acquire, align 8 | |
| ret <1 x i64> %ret | |
| } | |
| define void @store_atomic_vec1_i32(ptr %x, <1 x i32> %v) { | |
| ; CHECK-LABEL: store_atomic_vec1_i32: | |
| ; CHECK: # %bb.0: | |
| ; CHECK-NEXT: movl %esi, (%rdi) | |
| ; CHECK-NEXT: retq | |
| store atomic <1 x i32> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec1_i8(ptr %x, <1 x i8> %v) { | |
| ; CHECK-O3-LABEL: store_atomic_vec1_i8: | |
| ; CHECK-O3: # %bb.0: | |
| ; CHECK-O3-NEXT: movb %sil, (%rdi) | |
| ; CHECK-O3-NEXT: retq | |
| ; | |
| ; CHECK-O0-LABEL: store_atomic_vec1_i8: | |
| ; CHECK-O0: # %bb.0: | |
| ; CHECK-O0-NEXT: movb %sil, %al | |
| ; CHECK-O0-NEXT: movb %al, (%rdi) | |
| ; CHECK-O0-NEXT: retq | |
| store atomic <1 x i8> %v, ptr %x release, align 1 | |
| ret void | |
| } | |
| define void @store_atomic_vec1_i16(ptr %x, <1 x i16> %v) { | |
| ; CHECK-O3-LABEL: store_atomic_vec1_i16: | |
| ; CHECK-O3: # %bb.0: | |
| ; CHECK-O3-NEXT: movw %si, (%rdi) | |
| ; CHECK-O3-NEXT: retq | |
| ; | |
| ; CHECK-O0-LABEL: store_atomic_vec1_i16: | |
| ; CHECK-O0: # %bb.0: | |
| ; CHECK-O0-NEXT: movw %si, %ax | |
| ; CHECK-O0-NEXT: movw %ax, (%rdi) | |
| ; CHECK-O0-NEXT: retq | |
| store atomic <1 x i16> %v, ptr %x release, align 2 | |
| ret void | |
| } | |
| define void @store_atomic_vec1_i64(ptr %x, <1 x i64> %v) nounwind { | |
| ; CHECK-LABEL: store_atomic_vec1_i64: | |
| ; CHECK: # %bb.0: | |
| ; CHECK-NEXT: movq %rsi, (%rdi) | |
| ; CHECK-NEXT: retq | |
| store atomic <1 x i64> %v, ptr %x release, align 8 | |
| ret void | |
| } | |
| define void @store_atomic_vec1_ptr(ptr %x, <1 x ptr> %v) nounwind { | |
| ; CHECK-LABEL: store_atomic_vec1_ptr: | |
| ; CHECK: # %bb.0: | |
| ; CHECK-NEXT: movq %rsi, (%rdi) | |
| ; CHECK-NEXT: retq | |
| store atomic <1 x ptr> %v, ptr %x release, align 8 | |
| ret void | |
| } | |
| define void @store_atomic_vec1_bfloat(ptr %x, <1 x bfloat> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec1_bfloat: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: pextrw $0, %xmm0, %eax | |
| ; CHECK-SSE-O3-NEXT: movw %ax, (%rdi) | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec1_bfloat: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vpextrw $0, %xmm0, %eax | |
| ; CHECK-AVX-O3-NEXT: movw %ax, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec1_bfloat: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: pslld $16, %xmm0 | |
| ; CHECK-SSE-O0-NEXT: movd %xmm0, %ecx | |
| ; CHECK-SSE-O0-NEXT: btl $16, %ecx | |
| ; CHECK-SSE-O0-NEXT: movl %ecx, %eax | |
| ; CHECK-SSE-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF | |
| ; CHECK-SSE-O0-NEXT: orl $4194304, %ecx # imm = 0x400000 | |
| ; CHECK-SSE-O0-NEXT: ucomiss %xmm0, %xmm0 | |
| ; CHECK-SSE-O0-NEXT: cmovpl %ecx, %eax | |
| ; CHECK-SSE-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE-O0-NEXT: movw %ax, (%rdi) | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec1_bfloat: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vpslld $16, %xmm0, %xmm0 | |
| ; CHECK-AVX-O0-NEXT: vmovd %xmm0, %ecx | |
| ; CHECK-AVX-O0-NEXT: btl $16, %ecx | |
| ; CHECK-AVX-O0-NEXT: movl %ecx, %eax | |
| ; CHECK-AVX-O0-NEXT: adcl $32767, %eax # imm = 0x7FFF | |
| ; CHECK-AVX-O0-NEXT: orl $4194304, %ecx # imm = 0x400000 | |
| ; CHECK-AVX-O0-NEXT: vucomiss %xmm0, %xmm0 | |
| ; CHECK-AVX-O0-NEXT: cmovpl %ecx, %eax | |
| ; CHECK-AVX-O0-NEXT: shrl $16, %eax | |
| ; CHECK-AVX-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-AVX-O0-NEXT: movw %ax, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <1 x bfloat> %v, ptr %x release, align 2 | |
| ret void | |
| } | |
| define void @store_atomic_vec1_half(ptr %x, <1 x half> %v) { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec1_half: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: pextrw $0, %xmm0, %eax | |
| ; CHECK-SSE-O3-NEXT: movw %ax, (%rdi) | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec1_half: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vpextrw $0, %xmm0, %eax | |
| ; CHECK-AVX-O3-NEXT: movw %ax, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec1_half: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: pextrw $0, %xmm0, %eax | |
| ; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE-O0-NEXT: movw %ax, (%rdi) | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec1_half: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, %eax | |
| ; CHECK-AVX-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-AVX-O0-NEXT: movw %ax, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <1 x half> %v, ptr %x release, align 2 | |
| ret void | |
| } | |
| define void @store_atomic_vec1_float(ptr %x, <1 x float> %v) { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec1_float: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi) | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec1_float: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec1_float: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movss %xmm0, (%rdi) | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec1_float: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovss %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <1 x float> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec1_double_align(ptr %x, <1 x double> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec1_double_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movsd %xmm0, (%rdi) | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec1_double_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovsd %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec1_double_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movsd %xmm0, (%rdi) | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec1_double_align: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovsd %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <1 x double> %v, ptr %x release, align 8 | |
| ret void | |
| } | |
| define void @store_atomic_vec2_i8(ptr %x, <2 x i8> %v) { | |
| ; CHECK-SSE2-O3-LABEL: store_atomic_vec2_i8: | |
| ; CHECK-SSE2-O3: # %bb.0: | |
| ; CHECK-SSE2-O3-NEXT: movd %xmm0, %eax | |
| ; CHECK-SSE2-O3-NEXT: movw %ax, (%rdi) | |
| ; CHECK-SSE2-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O3-LABEL: store_atomic_vec2_i8: | |
| ; CHECK-SSE4-O3: # %bb.0: | |
| ; CHECK-SSE4-O3-NEXT: pextrw $0, %xmm0, (%rdi) | |
| ; CHECK-SSE4-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec2_i8: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vpextrw $0, %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE2-O0-LABEL: store_atomic_vec2_i8: | |
| ; CHECK-SSE2-O0: # %bb.0: | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, (%rdi) | |
| ; CHECK-SSE2-O0-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O0-LABEL: store_atomic_vec2_i8: | |
| ; CHECK-SSE4-O0: # %bb.0: | |
| ; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm0, (%rdi) | |
| ; CHECK-SSE4-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec2_i8: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <2 x i8> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec2_i16(ptr %x, <2 x i16> %v) { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec2_i16: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi) | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec2_i16: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec2_i16: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movd %xmm0, (%rdi) | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec2_i16: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <2 x i16> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec2_ptr270(ptr %x, <2 x ptr addrspace(270)> %v) { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec2_ptr270: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi) | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec2_ptr270: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec2_ptr270: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi) | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec2_ptr270: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <2 x ptr addrspace(270)> %v, ptr %x release, align 8 | |
| ret void | |
| } | |
| define void @store_atomic_vec2_i32_align(ptr %x, <2 x i32> %v) { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec2_i32_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi) | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec2_i32_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec2_i32_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi) | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec2_i32_align: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <2 x i32> %v, ptr %x release, align 8 | |
| ret void | |
| } | |
| define void @store_atomic_vec2_float_align(ptr %x, <2 x float> %v) { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec2_float_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi) | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec2_float_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec2_float_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi) | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec2_float_align: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <2 x float> %v, ptr %x release, align 8 | |
| ret void | |
| } | |
| define void @store_atomic_vec4_i8(ptr %x, <4 x i8> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec4_i8: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi) | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec4_i8: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec4_i8: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movd %xmm0, (%rdi) | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec4_i8: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <4 x i8> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec4_i16(ptr %x, <4 x i16> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec4_i16: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi) | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec4_i16: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec4_i16: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi) | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec4_i16: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <4 x i16> %v, ptr %x release, align 8 | |
| ret void | |
| } | |
| define <2 x i8> @atomic_vec2_i8(ptr %x) { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec2_i8: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movzwl (%rdi), %eax | |
| ; CHECK-SSE-O3-NEXT: movd %eax, %xmm0 | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec2_i8: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: movzwl (%rdi), %eax | |
| ; CHECK-AVX-O3-NEXT: vmovd %eax, %xmm0 | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec2_i8: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movw (%rdi), %cx | |
| ; CHECK-SSE-O0-NEXT: # implicit-def: $eax | |
| ; CHECK-SSE-O0-NEXT: movw %cx, %ax | |
| ; CHECK-SSE-O0-NEXT: movd %eax, %xmm0 | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec2_i8: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: movw (%rdi), %cx | |
| ; CHECK-AVX-O0-NEXT: # implicit-def: $eax | |
| ; CHECK-AVX-O0-NEXT: movw %cx, %ax | |
| ; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0 | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <2 x i8>, ptr %x acquire, align 4 | |
| ret <2 x i8> %ret | |
| } | |
| define <2 x i16> @atomic_vec2_i16(ptr %x) { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec2_i16: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec2_i16: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec2_i16: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec2_i16: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <2 x i16>, ptr %x acquire, align 4 | |
| ret <2 x i16> %ret | |
| } | |
| define <2 x ptr addrspace(270)> @atomic_vec2_ptr270(ptr %x) { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec2_ptr270: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movq (%rdi), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec2_ptr270: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovq (%rdi), %xmm0 | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec2_ptr270: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movq (%rdi), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec2_ptr270: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovq (%rdi), %xmm0 | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <2 x ptr addrspace(270)>, ptr %x acquire, align 8 | |
| ret <2 x ptr addrspace(270)> %ret | |
| } | |
| define <2 x ptr> @atomic_vec2_ptr_align(ptr %x) nounwind { | |
| ; CHECK-SSE2-O3-LABEL: atomic_vec2_ptr_align: | |
| ; CHECK-SSE2-O3: # %bb.0: | |
| ; CHECK-SSE2-O3-NEXT: pushq %rax | |
| ; CHECK-SSE2-O3-NEXT: movl $2, %esi | |
| ; CHECK-SSE2-O3-NEXT: callq __atomic_load_16@PLT | |
| ; CHECK-SSE2-O3-NEXT: movq %rdx, %xmm1 | |
| ; CHECK-SSE2-O3-NEXT: movq %rax, %xmm0 | |
| ; CHECK-SSE2-O3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] | |
| ; CHECK-SSE2-O3-NEXT: popq %rax | |
| ; CHECK-SSE2-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O3-LABEL: atomic_vec2_ptr_align: | |
| ; CHECK-SSE4-O3: # %bb.0: | |
| ; CHECK-SSE4-O3-NEXT: movaps (%rdi), %xmm0 | |
| ; CHECK-SSE4-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec2_ptr_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovaps (%rdi), %xmm0 | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE2-O0-LABEL: atomic_vec2_ptr_align: | |
| ; CHECK-SSE2-O0: # %bb.0: | |
| ; CHECK-SSE2-O0-NEXT: pushq %rax | |
| ; CHECK-SSE2-O0-NEXT: movl $2, %esi | |
| ; CHECK-SSE2-O0-NEXT: callq __atomic_load_16@PLT | |
| ; CHECK-SSE2-O0-NEXT: movq %rdx, %xmm1 | |
| ; CHECK-SSE2-O0-NEXT: movq %rax, %xmm0 | |
| ; CHECK-SSE2-O0-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] | |
| ; CHECK-SSE2-O0-NEXT: popq %rax | |
| ; CHECK-SSE2-O0-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O0-LABEL: atomic_vec2_ptr_align: | |
| ; CHECK-SSE4-O0: # %bb.0: | |
| ; CHECK-SSE4-O0-NEXT: movapd (%rdi), %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec2_ptr_align: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovapd (%rdi), %xmm0 | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <2 x ptr>, ptr %x acquire, align 16 | |
| ret <2 x ptr> %ret | |
| } | |
| define void @store_atomic_vec2_ptr_align(ptr %x, <2 x ptr> %v) nounwind { | |
| ; CHECK-SSE2-O3-LABEL: store_atomic_vec2_ptr_align: | |
| ; CHECK-SSE2-O3: # %bb.0: | |
| ; CHECK-SSE2-O3-NEXT: pushq %rax | |
| ; CHECK-SSE2-O3-NEXT: movq %xmm0, %rsi | |
| ; CHECK-SSE2-O3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] | |
| ; CHECK-SSE2-O3-NEXT: movq %xmm0, %rdx | |
| ; CHECK-SSE2-O3-NEXT: movl $3, %ecx | |
| ; CHECK-SSE2-O3-NEXT: callq __atomic_store_16@PLT | |
| ; CHECK-SSE2-O3-NEXT: popq %rax | |
| ; CHECK-SSE2-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O3-LABEL: store_atomic_vec2_ptr_align: | |
| ; CHECK-SSE4-O3: # %bb.0: | |
| ; CHECK-SSE4-O3-NEXT: movaps %xmm0, (%rdi) | |
| ; CHECK-SSE4-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec2_ptr_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovaps %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE2-O0-LABEL: store_atomic_vec2_ptr_align: | |
| ; CHECK-SSE2-O0: # %bb.0: | |
| ; CHECK-SSE2-O0-NEXT: pushq %rax | |
| ; CHECK-SSE2-O0-NEXT: movq %xmm0, %rsi | |
| ; CHECK-SSE2-O0-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] | |
| ; CHECK-SSE2-O0-NEXT: movq %xmm0, %rdx | |
| ; CHECK-SSE2-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE2-O0-NEXT: callq __atomic_store_16@PLT | |
| ; CHECK-SSE2-O0-NEXT: popq %rax | |
| ; CHECK-SSE2-O0-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O0-LABEL: store_atomic_vec2_ptr_align: | |
| ; CHECK-SSE4-O0: # %bb.0: | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm0, (%rdi) | |
| ; CHECK-SSE4-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec2_ptr_align: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovdqa %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <2 x ptr> %v, ptr %x release, align 16 | |
| ret void | |
| } | |
| define <4 x ptr addrspace(270)> @atomic_vec4_ptr270(ptr %x) nounwind { | |
| ; CHECK-SSE2-O3-LABEL: atomic_vec4_ptr270: | |
| ; CHECK-SSE2-O3: # %bb.0: | |
| ; CHECK-SSE2-O3-NEXT: pushq %rax | |
| ; CHECK-SSE2-O3-NEXT: movl $2, %esi | |
| ; CHECK-SSE2-O3-NEXT: callq __atomic_load_16@PLT | |
| ; CHECK-SSE2-O3-NEXT: movq %rdx, %xmm1 | |
| ; CHECK-SSE2-O3-NEXT: movq %rax, %xmm0 | |
| ; CHECK-SSE2-O3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] | |
| ; CHECK-SSE2-O3-NEXT: popq %rax | |
| ; CHECK-SSE2-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O3-LABEL: atomic_vec4_ptr270: | |
| ; CHECK-SSE4-O3: # %bb.0: | |
| ; CHECK-SSE4-O3-NEXT: movaps (%rdi), %xmm0 | |
| ; CHECK-SSE4-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec4_ptr270: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovaps (%rdi), %xmm0 | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE2-O0-LABEL: atomic_vec4_ptr270: | |
| ; CHECK-SSE2-O0: # %bb.0: | |
| ; CHECK-SSE2-O0-NEXT: pushq %rax | |
| ; CHECK-SSE2-O0-NEXT: movl $2, %esi | |
| ; CHECK-SSE2-O0-NEXT: callq __atomic_load_16@PLT | |
| ; CHECK-SSE2-O0-NEXT: movq %rdx, %xmm1 | |
| ; CHECK-SSE2-O0-NEXT: movq %rax, %xmm0 | |
| ; CHECK-SSE2-O0-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] | |
| ; CHECK-SSE2-O0-NEXT: popq %rax | |
| ; CHECK-SSE2-O0-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O0-LABEL: atomic_vec4_ptr270: | |
| ; CHECK-SSE4-O0: # %bb.0: | |
| ; CHECK-SSE4-O0-NEXT: movapd (%rdi), %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec4_ptr270: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovapd (%rdi), %xmm0 | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <4 x ptr addrspace(270)>, ptr %x acquire, align 16 | |
| ret <4 x ptr addrspace(270)> %ret | |
| } | |
| define void @store_atomic_vec4_ptr270_align(ptr %x, <4 x ptr addrspace(270)> %v) nounwind { | |
| ; CHECK-SSE2-O3-LABEL: store_atomic_vec4_ptr270_align: | |
| ; CHECK-SSE2-O3: # %bb.0: | |
| ; CHECK-SSE2-O3-NEXT: pushq %rax | |
| ; CHECK-SSE2-O3-NEXT: movq %xmm0, %rsi | |
| ; CHECK-SSE2-O3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] | |
| ; CHECK-SSE2-O3-NEXT: movq %xmm0, %rdx | |
| ; CHECK-SSE2-O3-NEXT: movl $3, %ecx | |
| ; CHECK-SSE2-O3-NEXT: callq __atomic_store_16@PLT | |
| ; CHECK-SSE2-O3-NEXT: popq %rax | |
| ; CHECK-SSE2-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O3-LABEL: store_atomic_vec4_ptr270_align: | |
| ; CHECK-SSE4-O3: # %bb.0: | |
| ; CHECK-SSE4-O3-NEXT: movaps %xmm0, (%rdi) | |
| ; CHECK-SSE4-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec4_ptr270_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovaps %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE2-O0-LABEL: store_atomic_vec4_ptr270_align: | |
| ; CHECK-SSE2-O0: # %bb.0: | |
| ; CHECK-SSE2-O0-NEXT: pushq %rax | |
| ; CHECK-SSE2-O0-NEXT: movq %xmm0, %rsi | |
| ; CHECK-SSE2-O0-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] | |
| ; CHECK-SSE2-O0-NEXT: movq %xmm0, %rdx | |
| ; CHECK-SSE2-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE2-O0-NEXT: callq __atomic_store_16@PLT | |
| ; CHECK-SSE2-O0-NEXT: popq %rax | |
| ; CHECK-SSE2-O0-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O0-LABEL: store_atomic_vec4_ptr270_align: | |
| ; CHECK-SSE4-O0: # %bb.0: | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm0, (%rdi) | |
| ; CHECK-SSE4-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec4_ptr270_align: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovdqa %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <4 x ptr addrspace(270)> %v, ptr %x release, align 16 | |
| ret void | |
| } | |
| define <2 x i32> @atomic_vec2_i32_align(ptr %x) { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec2_i32_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movq (%rdi), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec2_i32_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovq (%rdi), %xmm0 | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec2_i32_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movq (%rdi), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec2_i32_align: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovq (%rdi), %xmm0 | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <2 x i32>, ptr %x acquire, align 8 | |
| ret <2 x i32> %ret | |
| } | |
| define <2 x float> @atomic_vec2_float_align(ptr %x) { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec2_float_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movq (%rdi), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec2_float_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovq (%rdi), %xmm0 | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec2_float_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movq (%rdi), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec2_float_align: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovq (%rdi), %xmm0 | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <2 x float>, ptr %x acquire, align 8 | |
| ret <2 x float> %ret | |
| } | |
| define void @store_atomic_vec2_half(ptr %x, <2 x half> %v) { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec2_half: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi) | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec2_half: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec2_half: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm0, %xmm1 | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm1, %xmm0 | |
| ; CHECK-SSE-O0-NEXT: psrld $16, %xmm1 | |
| ; CHECK-SSE-O0-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] | |
| ; CHECK-SSE-O0-NEXT: movd %xmm0, (%rdi) | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec2_half: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <2 x half> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec2_bfloat(ptr %x, <2 x bfloat> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec2_bfloat: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi) | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec2_bfloat: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE2-O0-LABEL: store_atomic_vec2_bfloat: | |
| ; CHECK-SSE2-O0: # %bb.0: | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm0, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0 | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm1, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movl -{{[0-9]+}}(%rsp), %eax | |
| ; CHECK-SSE2-O0-NEXT: movl %eax, (%rdi) | |
| ; CHECK-SSE2-O0-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O0-LABEL: store_atomic_vec2_bfloat: | |
| ; CHECK-SSE4-O0: # %bb.0: | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1 | |
| ; CHECK-SSE4-O0-NEXT: xorps %xmm2, %xmm2 | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4],xmm0[5],xmm2[6],xmm0[7] | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1 | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm1, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm0, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movl -{{[0-9]+}}(%rsp), %eax | |
| ; CHECK-SSE4-O0-NEXT: movl %eax, (%rdi) | |
| ; CHECK-SSE4-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec2_bfloat: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <2 x bfloat> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec4_half(ptr %x, <4 x half> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec4_half: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi) | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec4_half: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE2-O0-LABEL: store_atomic_vec4_half: | |
| ; CHECK-SSE2-O0: # %bb.0: | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm3 | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm3, %xmm0 | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm3, %xmm2 | |
| ; CHECK-SSE2-O0-NEXT: psrlq $48, %xmm2 | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm3, %xmm1 | |
| ; CHECK-SSE2-O0-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1,1,1] | |
| ; CHECK-SSE2-O0-NEXT: psrld $16, %xmm3 | |
| ; CHECK-SSE2-O0-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3] | |
| ; CHECK-SSE2-O0-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] | |
| ; CHECK-SSE2-O0-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] | |
| ; CHECK-SSE2-O0-NEXT: movq %xmm0, (%rdi) | |
| ; CHECK-SSE2-O0-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O0-LABEL: store_atomic_vec4_half: | |
| ; CHECK-SSE4-O0: # %bb.0: | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm3 | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm3, %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm3, %xmm2 | |
| ; CHECK-SSE4-O0-NEXT: psrlq $48, %xmm2 | |
| ; CHECK-SSE4-O0-NEXT: movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3] | |
| ; CHECK-SSE4-O0-NEXT: psrld $16, %xmm3 | |
| ; CHECK-SSE4-O0-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3] | |
| ; CHECK-SSE4-O0-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3] | |
| ; CHECK-SSE4-O0-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] | |
| ; CHECK-SSE4-O0-NEXT: movq %xmm0, (%rdi) | |
| ; CHECK-SSE4-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec4_half: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <4 x half> %v, ptr %x release, align 8 | |
| ret void | |
| } | |
| define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec4_bfloat: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi) | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec4_bfloat: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE2-O0-LABEL: store_atomic_vec4_bfloat: | |
| ; CHECK-SSE2-O0: # %bb.0: | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm2 | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm2, %xmm3 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm3 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm2, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm2, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm2, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2 | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm3, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm2, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm1, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movq -{{[0-9]+}}(%rsp), %rax | |
| ; CHECK-SSE2-O0-NEXT: movq %rax, (%rdi) | |
| ; CHECK-SSE2-O0-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O0-LABEL: store_atomic_vec4_bfloat: | |
| ; CHECK-SSE4-O0: # %bb.0: | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1 | |
| ; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm2 | |
| ; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7] | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm3 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm3 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm1, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm1, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1 | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm3, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm2, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm1, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm0, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, -{{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movq -{{[0-9]+}}(%rsp), %rax | |
| ; CHECK-SSE4-O0-NEXT: movq %rax, (%rdi) | |
| ; CHECK-SSE4-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec4_bfloat: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <4 x bfloat> %v, ptr %x release, align 8 | |
| ret void | |
| } | |
| define void @store_atomic_vec4_float_align(ptr %x, <4 x float> %v) nounwind { | |
| ; CHECK-SSE2-O3-LABEL: store_atomic_vec4_float_align: | |
| ; CHECK-SSE2-O3: # %bb.0: | |
| ; CHECK-SSE2-O3-NEXT: pushq %rax | |
| ; CHECK-SSE2-O3-NEXT: movq %xmm0, %rsi | |
| ; CHECK-SSE2-O3-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1] | |
| ; CHECK-SSE2-O3-NEXT: movq %xmm0, %rdx | |
| ; CHECK-SSE2-O3-NEXT: movl $3, %ecx | |
| ; CHECK-SSE2-O3-NEXT: callq __atomic_store_16@PLT | |
| ; CHECK-SSE2-O3-NEXT: popq %rax | |
| ; CHECK-SSE2-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O3-LABEL: store_atomic_vec4_float_align: | |
| ; CHECK-SSE4-O3: # %bb.0: | |
| ; CHECK-SSE4-O3-NEXT: pushq %rbx | |
| ; CHECK-SSE4-O3-NEXT: movdqa (%rdi), %xmm1 | |
| ; CHECK-SSE4-O3-NEXT: pextrq $1, %xmm0, %rcx | |
| ; CHECK-SSE4-O3-NEXT: movq %xmm0, %rbx | |
| ; CHECK-SSE4-O3-NEXT: .p2align 4 | |
| ; CHECK-SSE4-O3-NEXT: .LBB41_1: # %atomicrmw.start | |
| ; CHECK-SSE4-O3-NEXT: # =>This Inner Loop Header: Depth=1 | |
| ; CHECK-SSE4-O3-NEXT: movq %xmm1, %rax | |
| ; CHECK-SSE4-O3-NEXT: pextrq $1, %xmm1, %rdx | |
| ; CHECK-SSE4-O3-NEXT: lock cmpxchg16b (%rdi) | |
| ; CHECK-SSE4-O3-NEXT: movq %rdx, %xmm0 | |
| ; CHECK-SSE4-O3-NEXT: movq %rax, %xmm1 | |
| ; CHECK-SSE4-O3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] | |
| ; CHECK-SSE4-O3-NEXT: jne .LBB41_1 | |
| ; CHECK-SSE4-O3-NEXT: # %bb.2: # %atomicrmw.end | |
| ; CHECK-SSE4-O3-NEXT: popq %rbx | |
| ; CHECK-SSE4-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec4_float_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovaps %xmm0, (%rdi) | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE2-O0-LABEL: store_atomic_vec4_float_align: | |
| ; CHECK-SSE2-O0: # %bb.0: | |
| ; CHECK-SSE2-O0-NEXT: pushq %rax | |
| ; CHECK-SSE2-O0-NEXT: movq %xmm0, %rsi | |
| ; CHECK-SSE2-O0-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1,1] | |
| ; CHECK-SSE2-O0-NEXT: movq %xmm0, %rdx | |
| ; CHECK-SSE2-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE2-O0-NEXT: callq __atomic_store_16@PLT | |
| ; CHECK-SSE2-O0-NEXT: popq %rax | |
| ; CHECK-SSE2-O0-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O0-LABEL: store_atomic_vec4_float_align: | |
| ; CHECK-SSE4-O0: # %bb.0: | |
| ; CHECK-SSE4-O0-NEXT: pushq %rbx | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill | |
| ; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill | |
| ; CHECK-SSE4-O0-NEXT: movaps (%rdi), %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill | |
| ; CHECK-SSE4-O0-NEXT: .LBB41_1: # %atomicrmw.start | |
| ; CHECK-SSE4-O0-NEXT: # =>This Inner Loop Header: Depth=1 | |
| ; CHECK-SSE4-O0-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload | |
| ; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload | |
| ; CHECK-SSE4-O0-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload | |
| ; CHECK-SSE4-O0-NEXT: pextrq $1, %xmm1, %rcx | |
| ; CHECK-SSE4-O0-NEXT: movq %xmm1, %rbx | |
| ; CHECK-SSE4-O0-NEXT: movq %xmm0, %rax | |
| ; CHECK-SSE4-O0-NEXT: pextrq $1, %xmm0, %rdx | |
| ; CHECK-SSE4-O0-NEXT: lock cmpxchg16b (%rsi) | |
| ; CHECK-SSE4-O0-NEXT: movq %rax, %rcx | |
| ; CHECK-SSE4-O0-NEXT: sete %al | |
| ; CHECK-SSE4-O0-NEXT: movq %rdx, %xmm1 | |
| ; CHECK-SSE4-O0-NEXT: movq %rcx, %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] | |
| ; CHECK-SSE4-O0-NEXT: testb $1, %al | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill | |
| ; CHECK-SSE4-O0-NEXT: jne .LBB41_2 | |
| ; CHECK-SSE4-O0-NEXT: jmp .LBB41_1 | |
| ; CHECK-SSE4-O0-NEXT: .LBB41_2: # %atomicrmw.end | |
| ; CHECK-SSE4-O0-NEXT: popq %rbx | |
| ; CHECK-SSE4-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec4_float_align: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovaps %xmm0, (%rdi) | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <4 x float> %v, ptr %x release, align 16 | |
| ret void | |
| } | |
| define <2 x half> @atomic_vec2_half(ptr %x) { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec2_half: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec2_half: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec2_half: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec2_half: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <2 x half>, ptr %x acquire, align 4 | |
| ret <2 x half> %ret | |
| } | |
| define <2 x bfloat> @atomic_vec2_bfloat(ptr %x) { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec2_bfloat: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec2_bfloat: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec2_bfloat: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec2_bfloat: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <2 x bfloat>, ptr %x acquire, align 4 | |
| ret <2 x bfloat> %ret | |
| } | |
| define <1 x ptr> @atomic_vec1_ptr(ptr %x) nounwind { | |
| ; CHECK-O3-LABEL: atomic_vec1_ptr: | |
| ; CHECK-O3: # %bb.0: | |
| ; CHECK-O3-NEXT: pushq %rax | |
| ; CHECK-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-O3-NEXT: movl $8, %edi | |
| ; CHECK-O3-NEXT: movl $2, %ecx | |
| ; CHECK-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-O3-NEXT: movq (%rsp), %rax | |
| ; CHECK-O3-NEXT: popq %rcx | |
| ; CHECK-O3-NEXT: retq | |
| ; | |
| ; CHECK-O0-LABEL: atomic_vec1_ptr: | |
| ; CHECK-O0: # %bb.0: | |
| ; CHECK-O0-NEXT: pushq %rax | |
| ; CHECK-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-O0-NEXT: movl $8, %edi | |
| ; CHECK-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-O0-NEXT: movl $2, %ecx | |
| ; CHECK-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-O0-NEXT: movq (%rsp), %rax | |
| ; CHECK-O0-NEXT: popq %rcx | |
| ; CHECK-O0-NEXT: retq | |
| %ret = load atomic <1 x ptr>, ptr %x acquire, align 4 | |
| ret <1 x ptr> %ret | |
| } | |
| define <1 x half> @atomic_vec1_half(ptr %x) { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec1_half: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movzwl (%rdi), %eax | |
| ; CHECK-SSE-O3-NEXT: movd %eax, %xmm0 | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec1_half: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: movzwl (%rdi), %eax | |
| ; CHECK-AVX-O3-NEXT: vmovd %eax, %xmm0 | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec1_half: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movw (%rdi), %cx | |
| ; CHECK-SSE-O0-NEXT: # implicit-def: $eax | |
| ; CHECK-SSE-O0-NEXT: movw %cx, %ax | |
| ; CHECK-SSE-O0-NEXT: movd %eax, %xmm0 | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec1_half: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: movw (%rdi), %cx | |
| ; CHECK-AVX-O0-NEXT: # implicit-def: $eax | |
| ; CHECK-AVX-O0-NEXT: movw %cx, %ax | |
| ; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0 | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <1 x half>, ptr %x acquire, align 2 | |
| ret <1 x half> %ret | |
| } | |
| define <1 x float> @atomic_vec1_float(ptr %x) { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec1_float: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec1_float: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec1_float: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec1_float: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <1 x float>, ptr %x acquire, align 4 | |
| ret <1 x float> %ret | |
| } | |
| define <1 x double> @atomic_vec1_double_align(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec1_double_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec1_double_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec1_double_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec1_double_align: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <1 x double>, ptr %x acquire, align 8 | |
| ret <1 x double> %ret | |
| } | |
| define <1 x i64> @atomic_vec1_i64(ptr %x) nounwind { | |
| ; CHECK-O3-LABEL: atomic_vec1_i64: | |
| ; CHECK-O3: # %bb.0: | |
| ; CHECK-O3-NEXT: pushq %rax | |
| ; CHECK-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-O3-NEXT: movl $8, %edi | |
| ; CHECK-O3-NEXT: movl $2, %ecx | |
| ; CHECK-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-O3-NEXT: movq (%rsp), %rax | |
| ; CHECK-O3-NEXT: popq %rcx | |
| ; CHECK-O3-NEXT: retq | |
| ; | |
| ; CHECK-O0-LABEL: atomic_vec1_i64: | |
| ; CHECK-O0: # %bb.0: | |
| ; CHECK-O0-NEXT: pushq %rax | |
| ; CHECK-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-O0-NEXT: movl $8, %edi | |
| ; CHECK-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-O0-NEXT: movl $2, %ecx | |
| ; CHECK-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-O0-NEXT: movq (%rsp), %rax | |
| ; CHECK-O0-NEXT: popq %rcx | |
| ; CHECK-O0-NEXT: retq | |
| %ret = load atomic <1 x i64>, ptr %x acquire, align 4 | |
| ret <1 x i64> %ret | |
| } | |
| define <1 x double> @atomic_vec1_double(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec1_double: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: pushq %rax | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $8, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O3-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero | |
| ; CHECK-SSE-O3-NEXT: popq %rax | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec1_double: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: pushq %rax | |
| ; CHECK-AVX-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O3-NEXT: movl $8, %edi | |
| ; CHECK-AVX-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX-O3-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero | |
| ; CHECK-AVX-O3-NEXT: popq %rax | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec1_double: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: pushq %rax | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movl $8, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O0-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero | |
| ; CHECK-SSE-O0-NEXT: popq %rax | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec1_double: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: pushq %rax | |
| ; CHECK-AVX-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O0-NEXT: movl $8, %edi | |
| ; CHECK-AVX-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX-O0-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero | |
| ; CHECK-AVX-O0-NEXT: popq %rax | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <1 x double>, ptr %x acquire, align 4 | |
| ret <1 x double> %ret | |
| } | |
| define <2 x i32> @atomic_vec2_i32(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec2_i32: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: pushq %rax | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $8, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O3-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero | |
| ; CHECK-SSE-O3-NEXT: popq %rax | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec2_i32: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: pushq %rax | |
| ; CHECK-AVX-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O3-NEXT: movl $8, %edi | |
| ; CHECK-AVX-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX-O3-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero | |
| ; CHECK-AVX-O3-NEXT: popq %rax | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec2_i32: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: pushq %rax | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movl $8, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O0-NEXT: movq {{.*#+}} xmm0 = mem[0],zero | |
| ; CHECK-SSE-O0-NEXT: popq %rax | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec2_i32: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: pushq %rax | |
| ; CHECK-AVX-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O0-NEXT: movl $8, %edi | |
| ; CHECK-AVX-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX-O0-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero | |
| ; CHECK-AVX-O0-NEXT: popq %rax | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <2 x i32>, ptr %x acquire, align 4 | |
| ret <2 x i32> %ret | |
| } | |
| define <4 x i8> @atomic_vec4_i8(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec4_i8: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec4_i8: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec4_i8: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec4_i8: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <4 x i8>, ptr %x acquire, align 4 | |
| ret <4 x i8> %ret | |
| } | |
| define <4 x i16> @atomic_vec4_i16(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec4_i16: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movq (%rdi), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec4_i16: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovq (%rdi), %xmm0 | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec4_i16: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movq (%rdi), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec4_i16: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovq (%rdi), %xmm0 | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <4 x i16>, ptr %x acquire, align 8 | |
| ret <4 x i16> %ret | |
| } | |
| define <4 x half> @atomic_vec4_half(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec4_half: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movq (%rdi), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec4_half: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovq (%rdi), %xmm0 | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec4_half: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movq (%rdi), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec4_half: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovq (%rdi), %xmm0 | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <4 x half>, ptr %x acquire, align 8 | |
| ret <4 x half> %ret | |
| } | |
| define <4 x bfloat> @atomic_vec4_bfloat(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec4_bfloat: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: movq (%rdi), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec4_bfloat: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovq (%rdi), %xmm0 | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec4_bfloat: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: movq (%rdi), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec4_bfloat: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovq (%rdi), %xmm0 | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <4 x bfloat>, ptr %x acquire, align 8 | |
| ret <4 x bfloat> %ret | |
| } | |
| define <4 x float> @atomic_vec4_float(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec4_float: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $24, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $16, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: addq $24, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec4_float: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: subq $24, %rsp | |
| ; CHECK-AVX-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O3-NEXT: movl $16, %edi | |
| ; CHECK-AVX-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX-O3-NEXT: vmovaps (%rsp), %xmm0 | |
| ; CHECK-AVX-O3-NEXT: addq $24, %rsp | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec4_float: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $24, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movl $16, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O0-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: addq $24, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec4_float: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: subq $24, %rsp | |
| ; CHECK-AVX-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O0-NEXT: movl $16, %edi | |
| ; CHECK-AVX-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX-O0-NEXT: vmovaps (%rsp), %xmm0 | |
| ; CHECK-AVX-O0-NEXT: addq $24, %rsp | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <4 x float>, ptr %x acquire, align 4 | |
| ret <4 x float> %ret | |
| } | |
| define <4 x float> @atomic_vec4_float_align(ptr %x) nounwind { | |
| ; CHECK-SSE2-O3-LABEL: atomic_vec4_float_align: | |
| ; CHECK-SSE2-O3: # %bb.0: | |
| ; CHECK-SSE2-O3-NEXT: pushq %rax | |
| ; CHECK-SSE2-O3-NEXT: movl $2, %esi | |
| ; CHECK-SSE2-O3-NEXT: callq __atomic_load_16@PLT | |
| ; CHECK-SSE2-O3-NEXT: movq %rdx, %xmm1 | |
| ; CHECK-SSE2-O3-NEXT: movq %rax, %xmm0 | |
| ; CHECK-SSE2-O3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] | |
| ; CHECK-SSE2-O3-NEXT: popq %rax | |
| ; CHECK-SSE2-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O3-LABEL: atomic_vec4_float_align: | |
| ; CHECK-SSE4-O3: # %bb.0: | |
| ; CHECK-SSE4-O3-NEXT: pushq %rbx | |
| ; CHECK-SSE4-O3-NEXT: xorl %eax, %eax | |
| ; CHECK-SSE4-O3-NEXT: xorl %edx, %edx | |
| ; CHECK-SSE4-O3-NEXT: xorl %ecx, %ecx | |
| ; CHECK-SSE4-O3-NEXT: xorl %ebx, %ebx | |
| ; CHECK-SSE4-O3-NEXT: lock cmpxchg16b (%rdi) | |
| ; CHECK-SSE4-O3-NEXT: movq %rdx, %xmm1 | |
| ; CHECK-SSE4-O3-NEXT: movq %rax, %xmm0 | |
| ; CHECK-SSE4-O3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] | |
| ; CHECK-SSE4-O3-NEXT: popq %rbx | |
| ; CHECK-SSE4-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec4_float_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: vmovaps (%rdi), %xmm0 | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE2-O0-LABEL: atomic_vec4_float_align: | |
| ; CHECK-SSE2-O0: # %bb.0: | |
| ; CHECK-SSE2-O0-NEXT: pushq %rax | |
| ; CHECK-SSE2-O0-NEXT: movl $2, %esi | |
| ; CHECK-SSE2-O0-NEXT: callq __atomic_load_16@PLT | |
| ; CHECK-SSE2-O0-NEXT: movq %rdx, %xmm1 | |
| ; CHECK-SSE2-O0-NEXT: movq %rax, %xmm0 | |
| ; CHECK-SSE2-O0-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] | |
| ; CHECK-SSE2-O0-NEXT: popq %rax | |
| ; CHECK-SSE2-O0-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O0-LABEL: atomic_vec4_float_align: | |
| ; CHECK-SSE4-O0: # %bb.0: | |
| ; CHECK-SSE4-O0-NEXT: pushq %rbx | |
| ; CHECK-SSE4-O0-NEXT: xorl %eax, %eax | |
| ; CHECK-SSE4-O0-NEXT: movl %eax, %ebx | |
| ; CHECK-SSE4-O0-NEXT: movq %rbx, %rax | |
| ; CHECK-SSE4-O0-NEXT: movq %rbx, %rdx | |
| ; CHECK-SSE4-O0-NEXT: movq %rbx, %rcx | |
| ; CHECK-SSE4-O0-NEXT: lock cmpxchg16b (%rdi) | |
| ; CHECK-SSE4-O0-NEXT: movq %rdx, %xmm1 | |
| ; CHECK-SSE4-O0-NEXT: movq %rax, %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] | |
| ; CHECK-SSE4-O0-NEXT: popq %rbx | |
| ; CHECK-SSE4-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec4_float_align: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: vmovaps (%rdi), %xmm0 | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <4 x float>, ptr %x acquire, align 16 | |
| ret <4 x float> %ret | |
| } | |
| define <8 x double> @atomic_vec8_double(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec8_double: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3 | |
| ; CHECK-SSE-O3-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O3-LABEL: atomic_vec8_double: | |
| ; CHECK-AVX2-O3: # %bb.0: | |
| ; CHECK-AVX2-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX2-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX2-O3-NEXT: vmovups (%rsp), %ymm0 | |
| ; CHECK-AVX2-O3-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1 | |
| ; CHECK-AVX2-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O3-LABEL: atomic_vec8_double: | |
| ; CHECK-AVX512-O3: # %bb.0: | |
| ; CHECK-AVX512-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX512-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX512-O3-NEXT: vmovups (%rsp), %zmm0 | |
| ; CHECK-AVX512-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec8_double: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O0-NEXT: movapd (%rsp), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: movapd {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O0-NEXT: movapd {{[0-9]+}}(%rsp), %xmm2 | |
| ; CHECK-SSE-O0-NEXT: movapd {{[0-9]+}}(%rsp), %xmm3 | |
| ; CHECK-SSE-O0-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O0-LABEL: atomic_vec8_double: | |
| ; CHECK-AVX2-O0: # %bb.0: | |
| ; CHECK-AVX2-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX2-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX2-O0-NEXT: vmovupd (%rsp), %ymm0 | |
| ; CHECK-AVX2-O0-NEXT: vmovupd {{[0-9]+}}(%rsp), %ymm1 | |
| ; CHECK-AVX2-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O0-LABEL: atomic_vec8_double: | |
| ; CHECK-AVX512-O0: # %bb.0: | |
| ; CHECK-AVX512-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX512-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX512-O0-NEXT: vmovupd (%rsp), %zmm0 | |
| ; CHECK-AVX512-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: retq | |
| %ret = load atomic <8 x double>, ptr %x acquire, align 4 | |
| ret <8 x double> %ret | |
| } | |
| define <16 x bfloat> @atomic_vec16_bfloat(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec16_bfloat: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $40, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $32, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O3-NEXT: addq $40, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec16_bfloat: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: subq $40, %rsp | |
| ; CHECK-AVX-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O3-NEXT: movl $32, %edi | |
| ; CHECK-AVX-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX-O3-NEXT: vmovups (%rsp), %ymm0 | |
| ; CHECK-AVX-O3-NEXT: addq $40, %rsp | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec16_bfloat: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $40, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movl $32, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O0-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O0-NEXT: addq $40, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec16_bfloat: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: subq $40, %rsp | |
| ; CHECK-AVX-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O0-NEXT: movl $32, %edi | |
| ; CHECK-AVX-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX-O0-NEXT: vmovups (%rsp), %ymm0 | |
| ; CHECK-AVX-O0-NEXT: addq $40, %rsp | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <16 x bfloat>, ptr %x acquire, align 4 | |
| ret <16 x bfloat> %ret | |
| } | |
| define <32 x half> @atomic_vec32_half(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec32_half: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3 | |
| ; CHECK-SSE-O3-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O3-LABEL: atomic_vec32_half: | |
| ; CHECK-AVX2-O3: # %bb.0: | |
| ; CHECK-AVX2-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX2-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX2-O3-NEXT: vmovups (%rsp), %ymm0 | |
| ; CHECK-AVX2-O3-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1 | |
| ; CHECK-AVX2-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O3-LABEL: atomic_vec32_half: | |
| ; CHECK-AVX512-O3: # %bb.0: | |
| ; CHECK-AVX512-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX512-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX512-O3-NEXT: vmovups (%rsp), %zmm0 | |
| ; CHECK-AVX512-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec32_half: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O0-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2 | |
| ; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3 | |
| ; CHECK-SSE-O0-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O0-LABEL: atomic_vec32_half: | |
| ; CHECK-AVX2-O0: # %bb.0: | |
| ; CHECK-AVX2-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX2-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX2-O0-NEXT: vmovups (%rsp), %ymm0 | |
| ; CHECK-AVX2-O0-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1 | |
| ; CHECK-AVX2-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O0-LABEL: atomic_vec32_half: | |
| ; CHECK-AVX512-O0: # %bb.0: | |
| ; CHECK-AVX512-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX512-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX512-O0-NEXT: vmovups (%rsp), %zmm0 | |
| ; CHECK-AVX512-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: retq | |
| %ret = load atomic <32 x half>, ptr %x acquire, align 4 | |
| ret <32 x half> %ret | |
| } | |
| define <8 x float> @atomic_vec8_float_align(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec8_float_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $40, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $32, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O3-NEXT: addq $40, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec8_float_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: subq $40, %rsp | |
| ; CHECK-AVX-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O3-NEXT: movl $32, %edi | |
| ; CHECK-AVX-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX-O3-NEXT: vmovups (%rsp), %ymm0 | |
| ; CHECK-AVX-O3-NEXT: addq $40, %rsp | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec8_float_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $40, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movl $32, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O0-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O0-NEXT: addq $40, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec8_float_align: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: subq $40, %rsp | |
| ; CHECK-AVX-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O0-NEXT: movl $32, %edi | |
| ; CHECK-AVX-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX-O0-NEXT: vmovups (%rsp), %ymm0 | |
| ; CHECK-AVX-O0-NEXT: addq $40, %rsp | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <8 x float>, ptr %x acquire, align 32 | |
| ret <8 x float> %ret | |
| } | |
| define <16 x float> @atomic_vec16_float_align(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec16_float_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3 | |
| ; CHECK-SSE-O3-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O3-LABEL: atomic_vec16_float_align: | |
| ; CHECK-AVX2-O3: # %bb.0: | |
| ; CHECK-AVX2-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX2-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX2-O3-NEXT: vmovups (%rsp), %ymm0 | |
| ; CHECK-AVX2-O3-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1 | |
| ; CHECK-AVX2-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O3-LABEL: atomic_vec16_float_align: | |
| ; CHECK-AVX512-O3: # %bb.0: | |
| ; CHECK-AVX512-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX512-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX512-O3-NEXT: vmovups (%rsp), %zmm0 | |
| ; CHECK-AVX512-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec16_float_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O0-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2 | |
| ; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3 | |
| ; CHECK-SSE-O0-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O0-LABEL: atomic_vec16_float_align: | |
| ; CHECK-AVX2-O0: # %bb.0: | |
| ; CHECK-AVX2-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX2-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX2-O0-NEXT: vmovups (%rsp), %ymm0 | |
| ; CHECK-AVX2-O0-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1 | |
| ; CHECK-AVX2-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O0-LABEL: atomic_vec16_float_align: | |
| ; CHECK-AVX512-O0: # %bb.0: | |
| ; CHECK-AVX512-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX512-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX512-O0-NEXT: vmovups (%rsp), %zmm0 | |
| ; CHECK-AVX512-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: retq | |
| %ret = load atomic <16 x float>, ptr %x acquire, align 64 | |
| ret <16 x float> %ret | |
| } | |
| define <8 x double> @atomic_vec8_double_align(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec8_double_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3 | |
| ; CHECK-SSE-O3-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O3-LABEL: atomic_vec8_double_align: | |
| ; CHECK-AVX2-O3: # %bb.0: | |
| ; CHECK-AVX2-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX2-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX2-O3-NEXT: vmovups (%rsp), %ymm0 | |
| ; CHECK-AVX2-O3-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1 | |
| ; CHECK-AVX2-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O3-LABEL: atomic_vec8_double_align: | |
| ; CHECK-AVX512-O3: # %bb.0: | |
| ; CHECK-AVX512-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX512-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX512-O3-NEXT: vmovups (%rsp), %zmm0 | |
| ; CHECK-AVX512-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec8_double_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O0-NEXT: movapd (%rsp), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: movapd {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O0-NEXT: movapd {{[0-9]+}}(%rsp), %xmm2 | |
| ; CHECK-SSE-O0-NEXT: movapd {{[0-9]+}}(%rsp), %xmm3 | |
| ; CHECK-SSE-O0-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O0-LABEL: atomic_vec8_double_align: | |
| ; CHECK-AVX2-O0: # %bb.0: | |
| ; CHECK-AVX2-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX2-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX2-O0-NEXT: vmovupd (%rsp), %ymm0 | |
| ; CHECK-AVX2-O0-NEXT: vmovupd {{[0-9]+}}(%rsp), %ymm1 | |
| ; CHECK-AVX2-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O0-LABEL: atomic_vec8_double_align: | |
| ; CHECK-AVX512-O0: # %bb.0: | |
| ; CHECK-AVX512-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX512-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX512-O0-NEXT: vmovupd (%rsp), %zmm0 | |
| ; CHECK-AVX512-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: retq | |
| %ret = load atomic <8 x double>, ptr %x acquire, align 64 | |
| ret <8 x double> %ret | |
| } | |
| define <16 x bfloat> @atomic_vec16_bfloat_align(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec16_bfloat_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $40, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $32, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O3-NEXT: addq $40, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: atomic_vec16_bfloat_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: subq $40, %rsp | |
| ; CHECK-AVX-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O3-NEXT: movl $32, %edi | |
| ; CHECK-AVX-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX-O3-NEXT: vmovups (%rsp), %ymm0 | |
| ; CHECK-AVX-O3-NEXT: addq $40, %rsp | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec16_bfloat_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $40, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movl $32, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O0-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O0-NEXT: addq $40, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: atomic_vec16_bfloat_align: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: subq $40, %rsp | |
| ; CHECK-AVX-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O0-NEXT: movl $32, %edi | |
| ; CHECK-AVX-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX-O0-NEXT: vmovups (%rsp), %ymm0 | |
| ; CHECK-AVX-O0-NEXT: addq $40, %rsp | |
| ; CHECK-AVX-O0-NEXT: retq | |
| %ret = load atomic <16 x bfloat>, ptr %x acquire, align 32 | |
| ret <16 x bfloat> %ret | |
| } | |
| define <32 x half> @atomic_vec32_half_align(ptr %x) nounwind { | |
| ; CHECK-SSE-O3-LABEL: atomic_vec32_half_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O3-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2 | |
| ; CHECK-SSE-O3-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3 | |
| ; CHECK-SSE-O3-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O3-LABEL: atomic_vec32_half_align: | |
| ; CHECK-AVX2-O3: # %bb.0: | |
| ; CHECK-AVX2-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX2-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX2-O3-NEXT: vmovups (%rsp), %ymm0 | |
| ; CHECK-AVX2-O3-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1 | |
| ; CHECK-AVX2-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O3-LABEL: atomic_vec32_half_align: | |
| ; CHECK-AVX512-O3: # %bb.0: | |
| ; CHECK-AVX512-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O3-NEXT: movl $2, %ecx | |
| ; CHECK-AVX512-O3-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX512-O3-NEXT: vmovups (%rsp), %zmm0 | |
| ; CHECK-AVX512-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: atomic_vec32_half_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $2, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-SSE-O0-NEXT: movaps (%rsp), %xmm0 | |
| ; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 | |
| ; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm2 | |
| ; CHECK-SSE-O0-NEXT: movaps {{[0-9]+}}(%rsp), %xmm3 | |
| ; CHECK-SSE-O0-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O0-LABEL: atomic_vec32_half_align: | |
| ; CHECK-AVX2-O0: # %bb.0: | |
| ; CHECK-AVX2-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX2-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX2-O0-NEXT: vmovups (%rsp), %ymm0 | |
| ; CHECK-AVX2-O0-NEXT: vmovups {{[0-9]+}}(%rsp), %ymm1 | |
| ; CHECK-AVX2-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O0-LABEL: atomic_vec32_half_align: | |
| ; CHECK-AVX512-O0: # %bb.0: | |
| ; CHECK-AVX512-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O0-NEXT: movl $2, %ecx | |
| ; CHECK-AVX512-O0-NEXT: callq __atomic_load@PLT | |
| ; CHECK-AVX512-O0-NEXT: vmovups (%rsp), %zmm0 | |
| ; CHECK-AVX512-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: retq | |
| %ret = load atomic <32 x half>, ptr %x acquire, align 64 | |
| ret <32 x half> %ret | |
| } | |
| define void @store_atomic_vec1_ptr_unaligned(ptr %x, <1 x ptr> %v) nounwind { | |
| ; CHECK-O3-LABEL: store_atomic_vec1_ptr_unaligned: | |
| ; CHECK-O3: # %bb.0: | |
| ; CHECK-O3-NEXT: pushq %rax | |
| ; CHECK-O3-NEXT: movq %rdi, %rax | |
| ; CHECK-O3-NEXT: movq %rsi, (%rsp) | |
| ; CHECK-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-O3-NEXT: movl $8, %edi | |
| ; CHECK-O3-NEXT: movq %rax, %rsi | |
| ; CHECK-O3-NEXT: movl $3, %ecx | |
| ; CHECK-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-O3-NEXT: popq %rax | |
| ; CHECK-O3-NEXT: retq | |
| ; | |
| ; CHECK-O0-LABEL: store_atomic_vec1_ptr_unaligned: | |
| ; CHECK-O0: # %bb.0: | |
| ; CHECK-O0-NEXT: pushq %rax | |
| ; CHECK-O0-NEXT: movq %rsi, %rax | |
| ; CHECK-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-O0-NEXT: movq %rax, (%rsp) | |
| ; CHECK-O0-NEXT: movl $8, %edi | |
| ; CHECK-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-O0-NEXT: movl $3, %ecx | |
| ; CHECK-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-O0-NEXT: popq %rax | |
| ; CHECK-O0-NEXT: retq | |
| store atomic <1 x ptr> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec1_i64_unaligned(ptr %x, <1 x i64> %v) nounwind { | |
| ; CHECK-O3-LABEL: store_atomic_vec1_i64_unaligned: | |
| ; CHECK-O3: # %bb.0: | |
| ; CHECK-O3-NEXT: pushq %rax | |
| ; CHECK-O3-NEXT: movq %rdi, %rax | |
| ; CHECK-O3-NEXT: movq %rsi, (%rsp) | |
| ; CHECK-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-O3-NEXT: movl $8, %edi | |
| ; CHECK-O3-NEXT: movq %rax, %rsi | |
| ; CHECK-O3-NEXT: movl $3, %ecx | |
| ; CHECK-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-O3-NEXT: popq %rax | |
| ; CHECK-O3-NEXT: retq | |
| ; | |
| ; CHECK-O0-LABEL: store_atomic_vec1_i64_unaligned: | |
| ; CHECK-O0: # %bb.0: | |
| ; CHECK-O0-NEXT: pushq %rax | |
| ; CHECK-O0-NEXT: movq %rsi, %rax | |
| ; CHECK-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-O0-NEXT: movq %rax, (%rsp) | |
| ; CHECK-O0-NEXT: movl $8, %edi | |
| ; CHECK-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-O0-NEXT: movl $3, %ecx | |
| ; CHECK-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-O0-NEXT: popq %rax | |
| ; CHECK-O0-NEXT: retq | |
| store atomic <1 x i64> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec1_double_unaligned(ptr %x, <1 x double> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec1_double_unaligned: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: pushq %rax | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movsd %xmm0, (%rsp) | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $8, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O3-NEXT: popq %rax | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec1_double_unaligned: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: pushq %rax | |
| ; CHECK-AVX-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O3-NEXT: vmovsd %xmm0, (%rsp) | |
| ; CHECK-AVX-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O3-NEXT: movl $8, %edi | |
| ; CHECK-AVX-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX-O3-NEXT: popq %rax | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec1_double_unaligned: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: pushq %rax | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movsd %xmm0, (%rsp) | |
| ; CHECK-SSE-O0-NEXT: movl $8, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O0-NEXT: popq %rax | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec1_double_unaligned: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: pushq %rax | |
| ; CHECK-AVX-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O0-NEXT: vmovsd %xmm0, (%rsp) | |
| ; CHECK-AVX-O0-NEXT: movl $8, %edi | |
| ; CHECK-AVX-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX-O0-NEXT: popq %rax | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <1 x double> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec2_i32_unaligned(ptr %x, <2 x i32> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec2_i32_unaligned: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: pushq %rax | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rsp) | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $8, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O3-NEXT: popq %rax | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec2_i32_unaligned: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: pushq %rax | |
| ; CHECK-AVX-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rsp) | |
| ; CHECK-AVX-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O3-NEXT: movl $8, %edi | |
| ; CHECK-AVX-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX-O3-NEXT: popq %rax | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec2_i32_unaligned: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: pushq %rax | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movq %xmm0, (%rsp) | |
| ; CHECK-SSE-O0-NEXT: movl $8, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O0-NEXT: popq %rax | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec2_i32_unaligned: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: pushq %rax | |
| ; CHECK-AVX-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rsp) | |
| ; CHECK-AVX-O0-NEXT: movl $8, %edi | |
| ; CHECK-AVX-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX-O0-NEXT: popq %rax | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <2 x i32> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec4_float_unaligned(ptr %x, <4 x float> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec4_float_unaligned: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $24, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp) | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $16, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O3-NEXT: addq $24, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec4_float_unaligned: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: subq $24, %rsp | |
| ; CHECK-AVX-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O3-NEXT: vmovaps %xmm0, (%rsp) | |
| ; CHECK-AVX-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O3-NEXT: movl $16, %edi | |
| ; CHECK-AVX-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX-O3-NEXT: addq $24, %rsp | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec4_float_unaligned: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $24, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm0, (%rsp) | |
| ; CHECK-SSE-O0-NEXT: movl $16, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O0-NEXT: addq $24, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec4_float_unaligned: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: subq $24, %rsp | |
| ; CHECK-AVX-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O0-NEXT: vmovaps %xmm0, (%rsp) | |
| ; CHECK-AVX-O0-NEXT: movl $16, %edi | |
| ; CHECK-AVX-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX-O0-NEXT: addq $24, %rsp | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <4 x float> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec8_double_unaligned(ptr %x, <8 x double> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec8_double_unaligned: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp) | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O3-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O3-LABEL: store_atomic_vec8_double_unaligned: | |
| ; CHECK-AVX2-O3: # %bb.0: | |
| ; CHECK-AVX2-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O3-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-AVX2-O3-NEXT: vmovups %ymm0, (%rsp) | |
| ; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX2-O3-NEXT: vzeroupper | |
| ; CHECK-AVX2-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX2-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O3-LABEL: store_atomic_vec8_double_unaligned: | |
| ; CHECK-AVX512-O3: # %bb.0: | |
| ; CHECK-AVX512-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O3-NEXT: vmovups %zmm0, (%rsp) | |
| ; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX512-O3-NEXT: vzeroupper | |
| ; CHECK-AVX512-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX512-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec8_double_unaligned: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movapd %xmm0, (%rsp) | |
| ; CHECK-SSE-O0-NEXT: movapd %xmm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movapd %xmm2, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movapd %xmm3, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O0-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O0-LABEL: store_atomic_vec8_double_unaligned: | |
| ; CHECK-AVX2-O0: # %bb.0: | |
| ; CHECK-AVX2-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O0-NEXT: vmovupd %ymm0, (%rsp) | |
| ; CHECK-AVX2-O0-NEXT: vmovupd %ymm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-AVX2-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX2-O0-NEXT: vzeroupper | |
| ; CHECK-AVX2-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX2-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O0-LABEL: store_atomic_vec8_double_unaligned: | |
| ; CHECK-AVX512-O0: # %bb.0: | |
| ; CHECK-AVX512-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O0-NEXT: vmovupd %zmm0, (%rsp) | |
| ; CHECK-AVX512-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX512-O0-NEXT: vzeroupper | |
| ; CHECK-AVX512-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX512-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: retq | |
| store atomic <8 x double> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec16_bfloat_unaligned(ptr %x, <16 x bfloat> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec16_bfloat_unaligned: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $40, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp) | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $32, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O3-NEXT: addq $40, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec16_bfloat_unaligned: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: subq $40, %rsp | |
| ; CHECK-AVX-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O3-NEXT: vmovups %ymm0, (%rsp) | |
| ; CHECK-AVX-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O3-NEXT: movl $32, %edi | |
| ; CHECK-AVX-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX-O3-NEXT: vzeroupper | |
| ; CHECK-AVX-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX-O3-NEXT: addq $40, %rsp | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE2-O0-LABEL: store_atomic_vec16_bfloat_unaligned: | |
| ; CHECK-SSE2-O0: # %bb.0: | |
| ; CHECK-SSE2-O0-NEXT: subq $40, %rsp | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm6 | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm14 | |
| ; CHECK-SSE2-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE2-O0-NEXT: xorps %xmm11, %xmm11 | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm11, %xmm3 | |
| ; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7] | |
| ; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm14[4],xmm11[5],xmm14[5],xmm11[6],xmm14[6],xmm11[7],xmm14[7] | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm6, %xmm7 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm7 | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm14, %xmm15 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm15 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm6, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm6, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm6, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm6, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm4 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm4 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm6, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm5 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm5 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm6, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm6 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm6 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm14, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm8 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm8 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm14, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm9 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm9 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm14, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm10 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm10 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm14, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm12 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm12 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm14, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm13 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm13 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm14, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm14 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm14 | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm15, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, (%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm14, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm13, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm12, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm11, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm10, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm9, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm8, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm7, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm6, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm5, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm4, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm3, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm2, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm1, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movl $32, %edi | |
| ; CHECK-SSE2-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE2-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE2-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE2-O0-NEXT: addq $40, %rsp | |
| ; CHECK-SSE2-O0-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O0-LABEL: store_atomic_vec16_bfloat_unaligned: | |
| ; CHECK-SSE4-O0: # %bb.0: | |
| ; CHECK-SSE4-O0-NEXT: subq $40, %rsp | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm5 | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm13 | |
| ; CHECK-SSE4-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm3 | |
| ; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7] | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm5, %xmm6 | |
| ; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm6 = xmm0[0],xmm6[1],xmm0[2],xmm6[3],xmm0[4],xmm6[5],xmm0[6],xmm6[7] | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm11 | |
| ; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm13[4],xmm11[5],xmm13[5],xmm11[6],xmm13[6],xmm11[7],xmm13[7] | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm13, %xmm14 | |
| ; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm14 = xmm0[0],xmm14[1],xmm0[2],xmm14[3],xmm0[4],xmm14[5],xmm0[6],xmm14[7] | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm5, %xmm7 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm7 | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm13, %xmm15 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm15 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm5, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm5, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm5, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm5, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm4 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm4 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm5, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm5 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm5 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm13, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm8 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm8 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm13, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm9 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm9 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm13, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm10 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm10 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm13, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm12 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm12 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm13, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm13 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm13 | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm15, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, (%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm14, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm13, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm12, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm11, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm10, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm9, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm8, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm7, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm6, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm5, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm4, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm3, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm2, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm1, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm0, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movl $32, %edi | |
| ; CHECK-SSE4-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE4-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE4-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE4-O0-NEXT: addq $40, %rsp | |
| ; CHECK-SSE4-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O0-LABEL: store_atomic_vec16_bfloat_unaligned: | |
| ; CHECK-AVX2-O0: # %bb.0: | |
| ; CHECK-AVX2-O0-NEXT: subq $40, %rsp | |
| ; CHECK-AVX2-O0-NEXT: vmovaps %ymm0, %ymm1 | |
| ; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm1, %xmm0 | |
| ; CHECK-AVX2-O0-NEXT: # kill: def $xmm1 killed $xmm1 killed $ymm1 | |
| ; CHECK-AVX2-O0-NEXT: vmovaps %xmm1, (%rsp) | |
| ; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, {{[0-9]+}}(%rsp) | |
| ; CHECK-AVX2-O0-NEXT: movl $32, %edi | |
| ; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX2-O0-NEXT: vzeroupper | |
| ; CHECK-AVX2-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX2-O0-NEXT: addq $40, %rsp | |
| ; CHECK-AVX2-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O0-LABEL: store_atomic_vec16_bfloat_unaligned: | |
| ; CHECK-AVX512-O0: # %bb.0: | |
| ; CHECK-AVX512-O0-NEXT: subq $40, %rsp | |
| ; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O0-NEXT: vmovups %ymm0, (%rsp) | |
| ; CHECK-AVX512-O0-NEXT: movl $32, %edi | |
| ; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX512-O0-NEXT: vzeroupper | |
| ; CHECK-AVX512-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX512-O0-NEXT: addq $40, %rsp | |
| ; CHECK-AVX512-O0-NEXT: retq | |
| store atomic <16 x bfloat> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec32_half_unaligned(ptr %x, <32 x half> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec32_half_unaligned: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp) | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O3-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O3-LABEL: store_atomic_vec32_half_unaligned: | |
| ; CHECK-AVX2-O3: # %bb.0: | |
| ; CHECK-AVX2-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O3-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-AVX2-O3-NEXT: vmovups %ymm0, (%rsp) | |
| ; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX2-O3-NEXT: vzeroupper | |
| ; CHECK-AVX2-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX2-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O3-LABEL: store_atomic_vec32_half_unaligned: | |
| ; CHECK-AVX512-O3: # %bb.0: | |
| ; CHECK-AVX512-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O3-NEXT: vmovups %zmm0, (%rsp) | |
| ; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX512-O3-NEXT: vzeroupper | |
| ; CHECK-AVX512-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX512-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec32_half_unaligned: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm0, (%rsp) | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O0-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O0-LABEL: store_atomic_vec32_half_unaligned: | |
| ; CHECK-AVX2-O0: # %bb.0: | |
| ; CHECK-AVX2-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O0-NEXT: vmovups %ymm0, (%rsp) | |
| ; CHECK-AVX2-O0-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-AVX2-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX2-O0-NEXT: vzeroupper | |
| ; CHECK-AVX2-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX2-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O0-LABEL: store_atomic_vec32_half_unaligned: | |
| ; CHECK-AVX512-O0: # %bb.0: | |
| ; CHECK-AVX512-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O0-NEXT: vmovups %zmm0, (%rsp) | |
| ; CHECK-AVX512-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX512-O0-NEXT: vzeroupper | |
| ; CHECK-AVX512-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX512-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: retq | |
| store atomic <32 x half> %v, ptr %x release, align 4 | |
| ret void | |
| } | |
| define void @store_atomic_vec8_float_align(ptr %x, <8 x float> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec8_float_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $40, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp) | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $32, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O3-NEXT: addq $40, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec8_float_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: subq $40, %rsp | |
| ; CHECK-AVX-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O3-NEXT: vmovups %ymm0, (%rsp) | |
| ; CHECK-AVX-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O3-NEXT: movl $32, %edi | |
| ; CHECK-AVX-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX-O3-NEXT: vzeroupper | |
| ; CHECK-AVX-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX-O3-NEXT: addq $40, %rsp | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec8_float_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $40, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm0, (%rsp) | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movl $32, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O0-NEXT: addq $40, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O0-LABEL: store_atomic_vec8_float_align: | |
| ; CHECK-AVX-O0: # %bb.0: | |
| ; CHECK-AVX-O0-NEXT: subq $40, %rsp | |
| ; CHECK-AVX-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O0-NEXT: vmovups %ymm0, (%rsp) | |
| ; CHECK-AVX-O0-NEXT: movl $32, %edi | |
| ; CHECK-AVX-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX-O0-NEXT: vzeroupper | |
| ; CHECK-AVX-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX-O0-NEXT: addq $40, %rsp | |
| ; CHECK-AVX-O0-NEXT: retq | |
| store atomic <8 x float> %v, ptr %x release, align 32 | |
| ret void | |
| } | |
| define void @store_atomic_vec16_float_align(ptr %x, <16 x float> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec16_float_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp) | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O3-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O3-LABEL: store_atomic_vec16_float_align: | |
| ; CHECK-AVX2-O3: # %bb.0: | |
| ; CHECK-AVX2-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O3-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-AVX2-O3-NEXT: vmovups %ymm0, (%rsp) | |
| ; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX2-O3-NEXT: vzeroupper | |
| ; CHECK-AVX2-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX2-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O3-LABEL: store_atomic_vec16_float_align: | |
| ; CHECK-AVX512-O3: # %bb.0: | |
| ; CHECK-AVX512-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O3-NEXT: vmovups %zmm0, (%rsp) | |
| ; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX512-O3-NEXT: vzeroupper | |
| ; CHECK-AVX512-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX512-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec16_float_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm0, (%rsp) | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O0-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O0-LABEL: store_atomic_vec16_float_align: | |
| ; CHECK-AVX2-O0: # %bb.0: | |
| ; CHECK-AVX2-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O0-NEXT: vmovups %ymm0, (%rsp) | |
| ; CHECK-AVX2-O0-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-AVX2-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX2-O0-NEXT: vzeroupper | |
| ; CHECK-AVX2-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX2-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O0-LABEL: store_atomic_vec16_float_align: | |
| ; CHECK-AVX512-O0: # %bb.0: | |
| ; CHECK-AVX512-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O0-NEXT: vmovups %zmm0, (%rsp) | |
| ; CHECK-AVX512-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX512-O0-NEXT: vzeroupper | |
| ; CHECK-AVX512-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX512-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: retq | |
| store atomic <16 x float> %v, ptr %x release, align 64 | |
| ret void | |
| } | |
| define void @store_atomic_vec8_double_align(ptr %x, <8 x double> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec8_double_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp) | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O3-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O3-LABEL: store_atomic_vec8_double_align: | |
| ; CHECK-AVX2-O3: # %bb.0: | |
| ; CHECK-AVX2-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O3-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-AVX2-O3-NEXT: vmovups %ymm0, (%rsp) | |
| ; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX2-O3-NEXT: vzeroupper | |
| ; CHECK-AVX2-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX2-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O3-LABEL: store_atomic_vec8_double_align: | |
| ; CHECK-AVX512-O3: # %bb.0: | |
| ; CHECK-AVX512-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O3-NEXT: vmovups %zmm0, (%rsp) | |
| ; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX512-O3-NEXT: vzeroupper | |
| ; CHECK-AVX512-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX512-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec8_double_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movapd %xmm0, (%rsp) | |
| ; CHECK-SSE-O0-NEXT: movapd %xmm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movapd %xmm2, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movapd %xmm3, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O0-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O0-LABEL: store_atomic_vec8_double_align: | |
| ; CHECK-AVX2-O0: # %bb.0: | |
| ; CHECK-AVX2-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O0-NEXT: vmovupd %ymm0, (%rsp) | |
| ; CHECK-AVX2-O0-NEXT: vmovupd %ymm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-AVX2-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX2-O0-NEXT: vzeroupper | |
| ; CHECK-AVX2-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX2-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O0-LABEL: store_atomic_vec8_double_align: | |
| ; CHECK-AVX512-O0: # %bb.0: | |
| ; CHECK-AVX512-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O0-NEXT: vmovupd %zmm0, (%rsp) | |
| ; CHECK-AVX512-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX512-O0-NEXT: vzeroupper | |
| ; CHECK-AVX512-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX512-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: retq | |
| store atomic <8 x double> %v, ptr %x release, align 64 | |
| ret void | |
| } | |
| define void @store_atomic_vec16_bfloat_align(ptr %x, <16 x bfloat> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec16_bfloat_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $40, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp) | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $32, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O3-NEXT: addq $40, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX-O3-LABEL: store_atomic_vec16_bfloat_align: | |
| ; CHECK-AVX-O3: # %bb.0: | |
| ; CHECK-AVX-O3-NEXT: subq $40, %rsp | |
| ; CHECK-AVX-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX-O3-NEXT: vmovups %ymm0, (%rsp) | |
| ; CHECK-AVX-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX-O3-NEXT: movl $32, %edi | |
| ; CHECK-AVX-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX-O3-NEXT: vzeroupper | |
| ; CHECK-AVX-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX-O3-NEXT: addq $40, %rsp | |
| ; CHECK-AVX-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE2-O0-LABEL: store_atomic_vec16_bfloat_align: | |
| ; CHECK-SSE2-O0: # %bb.0: | |
| ; CHECK-SSE2-O0-NEXT: subq $40, %rsp | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm6 | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm14 | |
| ; CHECK-SSE2-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE2-O0-NEXT: xorps %xmm11, %xmm11 | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm11, %xmm3 | |
| ; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7] | |
| ; CHECK-SSE2-O0-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm14[4],xmm11[5],xmm14[5],xmm11[6],xmm14[6],xmm11[7],xmm14[7] | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm6, %xmm7 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm7 | |
| ; CHECK-SSE2-O0-NEXT: movaps %xmm14, %xmm15 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm15 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm6, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm6, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm6, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm6, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm4 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm4 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm6, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm5 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm5 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm6, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm6 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm6 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $7, %xmm14, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm8 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm8 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $6, %xmm14, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm9 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm9 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $5, %xmm14, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm10 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm10 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm14, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm12 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm12 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm14, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm13 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm13 | |
| ; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm14, %eax | |
| ; CHECK-SSE2-O0-NEXT: movd %eax, %xmm14 | |
| ; CHECK-SSE2-O0-NEXT: pslld $16, %xmm14 | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm15, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, (%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm14, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm13, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm12, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm11, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm10, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm9, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm8, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm7, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm6, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm5, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm4, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm3, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm2, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm1, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax | |
| ; CHECK-SSE2-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE2-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE2-O0-NEXT: movl $32, %edi | |
| ; CHECK-SSE2-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE2-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE2-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE2-O0-NEXT: addq $40, %rsp | |
| ; CHECK-SSE2-O0-NEXT: retq | |
| ; | |
| ; CHECK-SSE4-O0-LABEL: store_atomic_vec16_bfloat_align: | |
| ; CHECK-SSE4-O0: # %bb.0: | |
| ; CHECK-SSE4-O0-NEXT: subq $40, %rsp | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm5 | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm13 | |
| ; CHECK-SSE4-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm3 | |
| ; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7] | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm5, %xmm6 | |
| ; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm6 = xmm0[0],xmm6[1],xmm0[2],xmm6[3],xmm0[4],xmm6[5],xmm0[6],xmm6[7] | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm11 | |
| ; CHECK-SSE4-O0-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm13[4],xmm11[5],xmm13[5],xmm11[6],xmm13[6],xmm11[7],xmm13[7] | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm13, %xmm14 | |
| ; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm14 = xmm0[0],xmm14[1],xmm0[2],xmm14[3],xmm0[4],xmm14[5],xmm0[6],xmm14[7] | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm5, %xmm7 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm7 | |
| ; CHECK-SSE4-O0-NEXT: movaps %xmm13, %xmm15 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm15 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm5, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm5, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm5, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm5, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm4 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm4 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm5, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm5 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm5 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $7, %xmm13, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm8 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm8 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $6, %xmm13, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm9 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm9 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $5, %xmm13, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm10 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm10 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm13, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm12 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm12 | |
| ; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm13, %eax | |
| ; CHECK-SSE4-O0-NEXT: movd %eax, %xmm13 | |
| ; CHECK-SSE4-O0-NEXT: pslld $16, %xmm13 | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm15, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, (%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm14, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm13, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm12, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm11, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm10, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm9, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm8, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm7, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm6, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm5, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm4, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm3, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm2, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm1, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movd %xmm0, %eax | |
| ; CHECK-SSE4-O0-NEXT: shrl $16, %eax | |
| ; CHECK-SSE4-O0-NEXT: # kill: def $ax killed $ax killed $eax | |
| ; CHECK-SSE4-O0-NEXT: movw %ax, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE4-O0-NEXT: movl $32, %edi | |
| ; CHECK-SSE4-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE4-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE4-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE4-O0-NEXT: addq $40, %rsp | |
| ; CHECK-SSE4-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O0-LABEL: store_atomic_vec16_bfloat_align: | |
| ; CHECK-AVX2-O0: # %bb.0: | |
| ; CHECK-AVX2-O0-NEXT: subq $40, %rsp | |
| ; CHECK-AVX2-O0-NEXT: vmovaps %ymm0, %ymm1 | |
| ; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O0-NEXT: vextracti128 $1, %ymm1, %xmm0 | |
| ; CHECK-AVX2-O0-NEXT: # kill: def $xmm1 killed $xmm1 killed $ymm1 | |
| ; CHECK-AVX2-O0-NEXT: vmovaps %xmm1, (%rsp) | |
| ; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, {{[0-9]+}}(%rsp) | |
| ; CHECK-AVX2-O0-NEXT: movl $32, %edi | |
| ; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX2-O0-NEXT: vzeroupper | |
| ; CHECK-AVX2-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX2-O0-NEXT: addq $40, %rsp | |
| ; CHECK-AVX2-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O0-LABEL: store_atomic_vec16_bfloat_align: | |
| ; CHECK-AVX512-O0: # %bb.0: | |
| ; CHECK-AVX512-O0-NEXT: subq $40, %rsp | |
| ; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O0-NEXT: vmovups %ymm0, (%rsp) | |
| ; CHECK-AVX512-O0-NEXT: movl $32, %edi | |
| ; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX512-O0-NEXT: vzeroupper | |
| ; CHECK-AVX512-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX512-O0-NEXT: addq $40, %rsp | |
| ; CHECK-AVX512-O0-NEXT: retq | |
| store atomic <16 x bfloat> %v, ptr %x release, align 32 | |
| ret void | |
| } | |
| define void @store_atomic_vec32_half_align(ptr %x, <32 x half> %v) nounwind { | |
| ; CHECK-SSE-O3-LABEL: store_atomic_vec32_half_align: | |
| ; CHECK-SSE-O3: # %bb.0: | |
| ; CHECK-SSE-O3-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O3-NEXT: movaps %xmm0, (%rsp) | |
| ; CHECK-SSE-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O3-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O3-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O3-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O3-LABEL: store_atomic_vec32_half_align: | |
| ; CHECK-AVX2-O3: # %bb.0: | |
| ; CHECK-AVX2-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O3-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-AVX2-O3-NEXT: vmovups %ymm0, (%rsp) | |
| ; CHECK-AVX2-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX2-O3-NEXT: vzeroupper | |
| ; CHECK-AVX2-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX2-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O3-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O3-LABEL: store_atomic_vec32_half_align: | |
| ; CHECK-AVX512-O3: # %bb.0: | |
| ; CHECK-AVX512-O3-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O3-NEXT: vmovups %zmm0, (%rsp) | |
| ; CHECK-AVX512-O3-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O3-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O3-NEXT: movl $3, %ecx | |
| ; CHECK-AVX512-O3-NEXT: vzeroupper | |
| ; CHECK-AVX512-O3-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX512-O3-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O3-NEXT: retq | |
| ; | |
| ; CHECK-SSE-O0-LABEL: store_atomic_vec32_half_align: | |
| ; CHECK-SSE-O0: # %bb.0: | |
| ; CHECK-SSE-O0-NEXT: subq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm0, (%rsp) | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm2, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movaps %xmm3, {{[0-9]+}}(%rsp) | |
| ; CHECK-SSE-O0-NEXT: movl $64, %edi | |
| ; CHECK-SSE-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-SSE-O0-NEXT: movl $3, %ecx | |
| ; CHECK-SSE-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-SSE-O0-NEXT: addq $72, %rsp | |
| ; CHECK-SSE-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX2-O0-LABEL: store_atomic_vec32_half_align: | |
| ; CHECK-AVX2-O0: # %bb.0: | |
| ; CHECK-AVX2-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX2-O0-NEXT: vmovups %ymm0, (%rsp) | |
| ; CHECK-AVX2-O0-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp) | |
| ; CHECK-AVX2-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX2-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX2-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX2-O0-NEXT: vzeroupper | |
| ; CHECK-AVX2-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX2-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX2-O0-NEXT: retq | |
| ; | |
| ; CHECK-AVX512-O0-LABEL: store_atomic_vec32_half_align: | |
| ; CHECK-AVX512-O0: # %bb.0: | |
| ; CHECK-AVX512-O0-NEXT: subq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: movq %rdi, %rsi | |
| ; CHECK-AVX512-O0-NEXT: vmovups %zmm0, (%rsp) | |
| ; CHECK-AVX512-O0-NEXT: movl $64, %edi | |
| ; CHECK-AVX512-O0-NEXT: movq %rsp, %rdx | |
| ; CHECK-AVX512-O0-NEXT: movl $3, %ecx | |
| ; CHECK-AVX512-O0-NEXT: vzeroupper | |
| ; CHECK-AVX512-O0-NEXT: callq __atomic_store@PLT | |
| ; CHECK-AVX512-O0-NEXT: addq $72, %rsp | |
| ; CHECK-AVX512-O0-NEXT: retq | |
| store atomic <32 x half> %v, ptr %x release, align 64 | |
| ret void | |
| } |