llvm/test/CodeGen/X86/atomic-load-store.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64    | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-SSE-O3,CHECK-SSE2-O3
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-SSE-O3,CHECK-SSE4-O3
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-AVX-O3,CHECK-AVX2-O3
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-AVX-O3,CHECK-AVX512-O3
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64    | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-SSE-O0,CHECK-SSE2-O0
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-SSE-O0,CHECK-SSE4-O0
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-AVX-O0,CHECK-AVX2-O0
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-AVX-O0,CHECK-AVX512-O0

define void @test1(ptr %ptr, i32 %val1) {
; CHECK-LABEL: test1:
; CHECK:       # %bb.0:
; CHECK-NEXT:    xchgl %esi, (%rdi)
; CHECK-NEXT:    retq
  store atomic i32 %val1, ptr %ptr seq_cst, align 4
  ret void
}

define void @test2(ptr %ptr, i32 %val1) {
; CHECK-LABEL: test2:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movl %esi, (%rdi)
; CHECK-NEXT:    retq
  store atomic i32 %val1, ptr %ptr release, align 4
  ret void
}

define i32 @test3(ptr %ptr) {
; CHECK-LABEL: test3:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movl (%rdi), %eax
; CHECK-NEXT:    retq
  %val = load atomic i32, ptr %ptr seq_cst, align 4
  ret i32 %val
}

define <1 x i32> @atomic_vec1_i32(ptr %x) {
; CHECK-LABEL: atomic_vec1_i32:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movl (%rdi), %eax
; CHECK-NEXT:    retq
  %ret = load atomic <1 x i32>, ptr %x acquire, align 4
  ret <1 x i32> %ret
}

define <1 x i8> @atomic_vec1_i8(ptr %x) {
; CHECK-O3-LABEL: atomic_vec1_i8:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    movzbl (%rdi), %eax
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: atomic_vec1_i8:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    movb (%rdi), %al
; CHECK-O0-NEXT:    retq
  %ret = load atomic <1 x i8>, ptr %x acquire, align 1
  ret <1 x i8> %ret
}

define <1 x i16> @atomic_vec1_i16(ptr %x) {
; CHECK-O3-LABEL: atomic_vec1_i16:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: atomic_vec1_i16:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    movw (%rdi), %ax
; CHECK-O0-NEXT:    retq
  %ret = load atomic <1 x i16>, ptr %x acquire, align 2
  ret <1 x i16> %ret
}

define <1 x i32> @atomic_vec1_i8_zext(ptr %x) {
; CHECK-O3-LABEL: atomic_vec1_i8_zext:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    movzbl (%rdi), %eax
; CHECK-O3-NEXT:    movzbl %al, %eax
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: atomic_vec1_i8_zext:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    movb (%rdi), %al
; CHECK-O0-NEXT:    movzbl %al, %eax
; CHECK-O0-NEXT:    retq
  %ret = load atomic <1 x i8>, ptr %x acquire, align 1
  %zret = zext <1 x i8> %ret to <1 x i32>
  ret <1 x i32> %zret
}

define <1 x i64> @atomic_vec1_i16_sext(ptr %x) {
; CHECK-O3-LABEL: atomic_vec1_i16_sext:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-O3-NEXT:    movswq %ax, %rax
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: atomic_vec1_i16_sext:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    movw (%rdi), %ax
; CHECK-O0-NEXT:    movswq %ax, %rax
; CHECK-O0-NEXT:    retq
  %ret = load atomic <1 x i16>, ptr %x acquire, align 2
  %sret = sext <1 x i16> %ret to <1 x i64>
  ret <1 x i64> %sret
}

define <1 x ptr addrspace(270)> @atomic_vec1_ptr270(ptr %x) {
; CHECK-LABEL: atomic_vec1_ptr270:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movl (%rdi), %eax
; CHECK-NEXT:    retq
  %ret = load atomic <1 x ptr addrspace(270)>, ptr %x acquire, align 4
  ret <1 x ptr addrspace(270)> %ret
}

define <1 x bfloat> @atomic_vec1_bfloat(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec1_bfloat:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-SSE-O3-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_bfloat:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-AVX-O3-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_bfloat:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movw (%rdi), %cx
; CHECK-SSE-O0-NEXT:    # implicit-def: $eax
; CHECK-SSE-O0-NEXT:    movw %cx, %ax
; CHECK-SSE-O0-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_bfloat:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    movw (%rdi), %cx
; CHECK-AVX-O0-NEXT:    # implicit-def: $eax
; CHECK-AVX-O0-NEXT:    movw %cx, %ax
; CHECK-AVX-O0-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <1 x bfloat>, ptr %x acquire, align 2
  ret <1 x bfloat> %ret
}

define <1 x ptr> @atomic_vec1_ptr_align(ptr %x) nounwind {
; CHECK-LABEL: atomic_vec1_ptr_align:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movq (%rdi), %rax
; CHECK-NEXT:    retq
  %ret = load atomic <1 x ptr>, ptr %x acquire, align 8
  ret <1 x ptr> %ret
}

define <1 x i64> @atomic_vec1_i64_align(ptr %x) nounwind {
; CHECK-LABEL: atomic_vec1_i64_align:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movq (%rdi), %rax
; CHECK-NEXT:    retq
  %ret = load atomic <1 x i64>, ptr %x acquire, align 8
  ret <1 x i64> %ret
}

define void @store_atomic_vec1_i32(ptr %x, <1 x i32> %v) {
; CHECK-LABEL: store_atomic_vec1_i32:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movl %esi, (%rdi)
; CHECK-NEXT:    retq
  store atomic <1 x i32> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec1_i8(ptr %x, <1 x i8> %v) {
; CHECK-O3-LABEL: store_atomic_vec1_i8:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    movb %sil, (%rdi)
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: store_atomic_vec1_i8:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    movb %sil, %al
; CHECK-O0-NEXT:    movb %al, (%rdi)
; CHECK-O0-NEXT:    retq
  store atomic <1 x i8> %v, ptr %x release, align 1
  ret void
}

define void @store_atomic_vec1_i16(ptr %x, <1 x i16> %v) {
; CHECK-O3-LABEL: store_atomic_vec1_i16:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    movw %si, (%rdi)
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: store_atomic_vec1_i16:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    movw %si, %ax
; CHECK-O0-NEXT:    movw %ax, (%rdi)
; CHECK-O0-NEXT:    retq
  store atomic <1 x i16> %v, ptr %x release, align 2
  ret void
}

define void @store_atomic_vec1_i64(ptr %x, <1 x i64> %v) nounwind {
; CHECK-LABEL: store_atomic_vec1_i64:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movq %rsi, (%rdi)
; CHECK-NEXT:    retq
  store atomic <1 x i64> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec1_ptr(ptr %x, <1 x ptr> %v) nounwind {
; CHECK-LABEL: store_atomic_vec1_ptr:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movq %rsi, (%rdi)
; CHECK-NEXT:    retq
  store atomic <1 x ptr> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec1_bfloat(ptr %x, <1 x bfloat> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_bfloat:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE-O3-NEXT:    movw %ax, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_bfloat:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vpextrw $0, %xmm0, %eax
; CHECK-AVX-O3-NEXT:    movw %ax, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_bfloat:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    pushq %rax
; CHECK-SSE-O0-NEXT:    movq %rdi, (%rsp) # 8-byte Spill
; CHECK-SSE-O0-NEXT:    pslld $16, %xmm0
; CHECK-SSE-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE-O0-NEXT:    movq (%rsp), %rdi # 8-byte Reload
; CHECK-SSE-O0-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE-O0-NEXT:    movw %ax, (%rdi)
; CHECK-SSE-O0-NEXT:    popq %rax
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_bfloat:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    pushq %rax
; CHECK-AVX-O0-NEXT:    movq %rdi, (%rsp) # 8-byte Spill
; CHECK-AVX-O0-NEXT:    vpslld $16, %xmm0, %xmm0
; CHECK-AVX-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX-O0-NEXT:    movq (%rsp), %rdi # 8-byte Reload
; CHECK-AVX-O0-NEXT:    vpextrw $0, %xmm0, %eax
; CHECK-AVX-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-AVX-O0-NEXT:    movw %ax, (%rdi)
; CHECK-AVX-O0-NEXT:    popq %rax
; CHECK-AVX-O0-NEXT:    retq
  store atomic <1 x bfloat> %v, ptr %x release, align 2
  ret void
}

define void @store_atomic_vec1_half(ptr %x, <1 x half> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_half:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE-O3-NEXT:    movw %ax, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_half:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vpextrw $0, %xmm0, %eax
; CHECK-AVX-O3-NEXT:    movw %ax, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_half:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE-O0-NEXT:    movw %ax, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_half:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vpextrw $0, %xmm0, %eax
; CHECK-AVX-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-AVX-O0-NEXT:    movw %ax, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <1 x half> %v, ptr %x release, align 2
  ret void
}

define void @store_atomic_vec1_float(ptr %x, <1 x float> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_float:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_float:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_float:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movss %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_float:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovss %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <1 x float> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec1_double_align(ptr %x, <1 x double> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_double_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movsd %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_double_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovsd %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_double_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movsd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_double_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovsd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <1 x double> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec2_i8(ptr %x, <2 x i8> %v) {
; CHECK-SSE2-O3-LABEL: store_atomic_vec2_i8:
; CHECK-SSE2-O3:       # %bb.0:
; CHECK-SSE2-O3-NEXT:    movd %xmm0, %eax
; CHECK-SSE2-O3-NEXT:    movw %ax, (%rdi)
; CHECK-SSE2-O3-NEXT:    retq
;
; CHECK-SSE4-O3-LABEL: store_atomic_vec2_i8:
; CHECK-SSE4-O3:       # %bb.0:
; CHECK-SSE4-O3-NEXT:    pextrw $0, %xmm0, (%rdi)
; CHECK-SSE4-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i8:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vpextrw $0, %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec2_i8:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    movd %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, (%rdi)
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec2_i8:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm0, (%rdi)
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i8:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vpextrw $0, %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x i8> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec2_i16(ptr %x, <2 x i16> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_i16:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i16:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_i16:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i16:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x i16> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec2_ptr270(ptr %x, <2 x ptr addrspace(270)> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_ptr270:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_ptr270:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_ptr270:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_ptr270:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x ptr addrspace(270)> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec2_i32_align(ptr %x, <2 x i32> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_i32_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i32_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_i32_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i32_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x i32> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec2_float_align(ptr %x, <2 x float> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_float_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_float_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_float_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_float_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x float> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec4_i8(ptr %x, <4 x i8> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_i8:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_i8:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec4_i8:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_i8:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <4 x i8> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec4_i16(ptr %x, <4 x i16> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_i16:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_i16:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec4_i16:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_i16:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <4 x i16> %v, ptr %x release, align 8
  ret void
}

define <2 x i8> @atomic_vec2_i8(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_i8:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-SSE-O3-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_i8:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-AVX-O3-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_i8:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movw (%rdi), %cx
; CHECK-SSE-O0-NEXT:    # implicit-def: $eax
; CHECK-SSE-O0-NEXT:    movw %cx, %ax
; CHECK-SSE-O0-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_i8:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    movw (%rdi), %cx
; CHECK-AVX-O0-NEXT:    # implicit-def: $eax
; CHECK-AVX-O0-NEXT:    movw %cx, %ax
; CHECK-AVX-O0-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x i8>, ptr %x acquire, align 4
  ret <2 x i8> %ret
}

define <2 x i16> @atomic_vec2_i16(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_i16:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_i16:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_i16:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_i16:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x i16>, ptr %x acquire, align 4
  ret <2 x i16> %ret
}

define <2 x ptr addrspace(270)> @atomic_vec2_ptr270(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_ptr270:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_ptr270:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_ptr270:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_ptr270:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x ptr addrspace(270)>, ptr %x acquire, align 8
  ret <2 x ptr addrspace(270)> %ret
}
define <2 x ptr> @atomic_vec2_ptr_align(ptr %x) nounwind {
; CHECK-SSE2-O3-LABEL: atomic_vec2_ptr_align:
; CHECK-SSE2-O3:       # %bb.0:
; CHECK-SSE2-O3-NEXT:    pushq %rax
; CHECK-SSE2-O3-NEXT:    movl $2, %esi
; CHECK-SSE2-O3-NEXT:    callq __atomic_load_16@PLT
; CHECK-SSE2-O3-NEXT:    movq %rdx, %xmm1
; CHECK-SSE2-O3-NEXT:    movq %rax, %xmm0
; CHECK-SSE2-O3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O3-NEXT:    popq %rax
; CHECK-SSE2-O3-NEXT:    retq
;
; CHECK-SSE4-O3-LABEL: atomic_vec2_ptr_align:
; CHECK-SSE4-O3:       # %bb.0:
; CHECK-SSE4-O3-NEXT:    movaps (%rdi), %xmm0
; CHECK-SSE4-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_ptr_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovaps (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: atomic_vec2_ptr_align:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    pushq %rax
; CHECK-SSE2-O0-NEXT:    movl $2, %esi
; CHECK-SSE2-O0-NEXT:    callq __atomic_load_16@PLT
; CHECK-SSE2-O0-NEXT:    movq %rdx, %xmm1
; CHECK-SSE2-O0-NEXT:    movq %rax, %xmm0
; CHECK-SSE2-O0-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O0-NEXT:    popq %rax
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: atomic_vec2_ptr_align:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    movapd (%rdi), %xmm0
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_ptr_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovapd (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x ptr>, ptr %x acquire, align 16
  ret <2 x ptr> %ret
}

define void @store_atomic_vec2_ptr_align(ptr %x, <2 x ptr> %v) nounwind {
; CHECK-SSE2-O3-LABEL: store_atomic_vec2_ptr_align:
; CHECK-SSE2-O3:       # %bb.0:
; CHECK-SSE2-O3-NEXT:    pushq %rax
; CHECK-SSE2-O3-NEXT:    movq %xmm0, %rsi
; CHECK-SSE2-O3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; CHECK-SSE2-O3-NEXT:    movq %xmm0, %rdx
; CHECK-SSE2-O3-NEXT:    movl $3, %ecx
; CHECK-SSE2-O3-NEXT:    callq __atomic_store_16@PLT
; CHECK-SSE2-O3-NEXT:    popq %rax
; CHECK-SSE2-O3-NEXT:    retq
;
; CHECK-SSE4-O3-LABEL: store_atomic_vec2_ptr_align:
; CHECK-SSE4-O3:       # %bb.0:
; CHECK-SSE4-O3-NEXT:    movaps %xmm0, (%rdi)
; CHECK-SSE4-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_ptr_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovaps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec2_ptr_align:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    pushq %rax
; CHECK-SSE2-O0-NEXT:    movq %xmm0, %rsi
; CHECK-SSE2-O0-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; CHECK-SSE2-O0-NEXT:    movq %xmm0, %rdx
; CHECK-SSE2-O0-NEXT:    movl $3, %ecx
; CHECK-SSE2-O0-NEXT:    callq __atomic_store_16@PLT
; CHECK-SSE2-O0-NEXT:    popq %rax
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec2_ptr_align:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, (%rdi)
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_ptr_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovdqa %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x ptr> %v, ptr %x release, align 16
  ret void
}

define <4 x ptr addrspace(270)> @atomic_vec4_ptr270(ptr %x) nounwind {
; CHECK-SSE2-O3-LABEL: atomic_vec4_ptr270:
; CHECK-SSE2-O3:       # %bb.0:
; CHECK-SSE2-O3-NEXT:    pushq %rax
; CHECK-SSE2-O3-NEXT:    movl $2, %esi
; CHECK-SSE2-O3-NEXT:    callq __atomic_load_16@PLT
; CHECK-SSE2-O3-NEXT:    movq %rdx, %xmm1
; CHECK-SSE2-O3-NEXT:    movq %rax, %xmm0
; CHECK-SSE2-O3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O3-NEXT:    popq %rax
; CHECK-SSE2-O3-NEXT:    retq
;
; CHECK-SSE4-O3-LABEL: atomic_vec4_ptr270:
; CHECK-SSE4-O3:       # %bb.0:
; CHECK-SSE4-O3-NEXT:    movaps (%rdi), %xmm0
; CHECK-SSE4-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_ptr270:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovaps (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: atomic_vec4_ptr270:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    pushq %rax
; CHECK-SSE2-O0-NEXT:    movl $2, %esi
; CHECK-SSE2-O0-NEXT:    callq __atomic_load_16@PLT
; CHECK-SSE2-O0-NEXT:    movq %rdx, %xmm1
; CHECK-SSE2-O0-NEXT:    movq %rax, %xmm0
; CHECK-SSE2-O0-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O0-NEXT:    popq %rax
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: atomic_vec4_ptr270:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    movapd (%rdi), %xmm0
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_ptr270:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovapd (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <4 x ptr addrspace(270)>, ptr %x acquire, align 16
  ret <4 x ptr addrspace(270)> %ret
}

define void @store_atomic_vec4_ptr270_align(ptr %x, <4 x ptr addrspace(270)> %v) nounwind {
; CHECK-SSE2-O3-LABEL: store_atomic_vec4_ptr270_align:
; CHECK-SSE2-O3:       # %bb.0:
; CHECK-SSE2-O3-NEXT:    pushq %rax
; CHECK-SSE2-O3-NEXT:    movq %xmm0, %rsi
; CHECK-SSE2-O3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; CHECK-SSE2-O3-NEXT:    movq %xmm0, %rdx
; CHECK-SSE2-O3-NEXT:    movl $3, %ecx
; CHECK-SSE2-O3-NEXT:    callq __atomic_store_16@PLT
; CHECK-SSE2-O3-NEXT:    popq %rax
; CHECK-SSE2-O3-NEXT:    retq
;
; CHECK-SSE4-O3-LABEL: store_atomic_vec4_ptr270_align:
; CHECK-SSE4-O3:       # %bb.0:
; CHECK-SSE4-O3-NEXT:    movaps %xmm0, (%rdi)
; CHECK-SSE4-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_ptr270_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovaps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec4_ptr270_align:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    pushq %rax
; CHECK-SSE2-O0-NEXT:    movq %xmm0, %rsi
; CHECK-SSE2-O0-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; CHECK-SSE2-O0-NEXT:    movq %xmm0, %rdx
; CHECK-SSE2-O0-NEXT:    movl $3, %ecx
; CHECK-SSE2-O0-NEXT:    callq __atomic_store_16@PLT
; CHECK-SSE2-O0-NEXT:    popq %rax
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec4_ptr270_align:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, (%rdi)
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_ptr270_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovdqa %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <4 x ptr addrspace(270)> %v, ptr %x release, align 16
  ret void
}

define <2 x i32> @atomic_vec2_i32_align(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_i32_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_i32_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_i32_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_i32_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x i32>, ptr %x acquire, align 8
  ret <2 x i32> %ret
}

define <2 x float> @atomic_vec2_float_align(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_float_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_float_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_float_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_float_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x float>, ptr %x acquire, align 8
  ret <2 x float> %ret
}

define void @store_atomic_vec2_half(ptr %x, <2 x half> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_half:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_half:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_half:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE-O0-NEXT:    movaps %xmm1, %xmm0
; CHECK-SSE-O0-NEXT:    psrld $16, %xmm1
; CHECK-SSE-O0-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; CHECK-SSE-O0-NEXT:    movd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_half:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x half> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec2_bfloat(ptr %x, <2 x bfloat> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_bfloat:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_bfloat:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec2_bfloat:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    subq $24, %rsp
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE2-O0-NEXT:    movaps %xmm1, %xmm0
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm0
; CHECK-SSE2-O0-NEXT:    pextrw $1, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm1
; CHECK-SSE2-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    movl {{[0-9]+}}(%rsp), %eax
; CHECK-SSE2-O0-NEXT:    movl %eax, (%rdi)
; CHECK-SSE2-O0-NEXT:    addq $24, %rsp
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec2_bfloat:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    subq $24, %rsp
; CHECK-SSE4-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE4-O0-NEXT:    xorps %xmm2, %xmm2
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm0
; CHECK-SSE4-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    movl {{[0-9]+}}(%rsp), %eax
; CHECK-SSE4-O0-NEXT:    movl %eax, (%rdi)
; CHECK-SSE4-O0-NEXT:    addq $24, %rsp
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec2_bfloat:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $24, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-AVX2-O0-NEXT:    vxorps %xmm1, %xmm1, %xmm1
; CHECK-AVX2-O0-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; CHECK-AVX2-O0-NEXT:    vpslld $16, %xmm0, %xmm0
; CHECK-AVX2-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    movl {{[0-9]+}}(%rsp), %eax
; CHECK-AVX2-O0-NEXT:    movl %eax, (%rdi)
; CHECK-AVX2-O0-NEXT:    addq $24, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec2_bfloat:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    vmovd %xmm0, (%rdi)
; CHECK-AVX512-O0-NEXT:    retq
  store atomic <2 x bfloat> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec4_half(ptr %x, <4 x half> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_half:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_half:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec4_half:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm3
; CHECK-SSE2-O0-NEXT:    movaps %xmm3, %xmm0
; CHECK-SSE2-O0-NEXT:    movaps %xmm3, %xmm2
; CHECK-SSE2-O0-NEXT:    psrlq $48, %xmm2
; CHECK-SSE2-O0-NEXT:    movaps %xmm3, %xmm1
; CHECK-SSE2-O0-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
; CHECK-SSE2-O0-NEXT:    psrld $16, %xmm3
; CHECK-SSE2-O0-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
; CHECK-SSE2-O0-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
; CHECK-SSE2-O0-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; CHECK-SSE2-O0-NEXT:    movq %xmm0, (%rdi)
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec4_half:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm3
; CHECK-SSE4-O0-NEXT:    movaps %xmm3, %xmm0
; CHECK-SSE4-O0-NEXT:    movaps %xmm3, %xmm2
; CHECK-SSE4-O0-NEXT:    psrlq $48, %xmm2
; CHECK-SSE4-O0-NEXT:    movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
; CHECK-SSE4-O0-NEXT:    psrld $16, %xmm3
; CHECK-SSE4-O0-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
; CHECK-SSE4-O0-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
; CHECK-SSE4-O0-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; CHECK-SSE4-O0-NEXT:    movq %xmm0, (%rdi)
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_half:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <4 x half> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_bfloat:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    subq $40, %rsp
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE2-O0-NEXT:    movaps %xmm1, %xmm0
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm0
; CHECK-SSE2-O0-NEXT:    pextrw $3, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm3
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm3
; CHECK-SSE2-O0-NEXT:    pextrw $2, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm2
; CHECK-SSE2-O0-NEXT:    pextrw $1, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm1
; CHECK-SSE2-O0-NEXT:    movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    movq {{[0-9]+}}(%rsp), %rax
; CHECK-SSE2-O0-NEXT:    movq %rax, (%rdi)
; CHECK-SSE2-O0-NEXT:    addq $40, %rsp
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    subq $40, %rsp
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE4-O0-NEXT:    xorps %xmm0, %xmm0
; CHECK-SSE4-O0-NEXT:    movaps %xmm1, %xmm3
; CHECK-SSE4-O0-NEXT:    pblendw {{.*#+}} xmm3 = xmm0[0],xmm3[1],xmm0[2],xmm3[3],xmm0[4],xmm3[5],xmm0[6],xmm3[7]
; CHECK-SSE4-O0-NEXT:    movaps %xmm1, %xmm0
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm0
; CHECK-SSE4-O0-NEXT:    pextrw $3, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm2
; CHECK-SSE4-O0-NEXT:    pextrw $2, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm1
; CHECK-SSE4-O0-NEXT:    movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    movq {{[0-9]+}}(%rsp), %rax
; CHECK-SSE4-O0-NEXT:    movq %rax, (%rdi)
; CHECK-SSE4-O0-NEXT:    addq $40, %rsp
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $40, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm1, %xmm0, %xmm3
; CHECK-AVX2-O0-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm1, %xmm0, %xmm2
; CHECK-AVX2-O0-NEXT:    vxorps %xmm1, %xmm1, %xmm1
; CHECK-AVX2-O0-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; CHECK-AVX2-O0-NEXT:    vpslld $16, %xmm0, %xmm0
; CHECK-AVX2-O0-NEXT:    vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    movq {{[0-9]+}}(%rsp), %rax
; CHECK-AVX2-O0-NEXT:    movq %rax, (%rdi)
; CHECK-AVX2-O0-NEXT:    addq $40, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    vmovq %xmm0, (%rdi)
; CHECK-AVX512-O0-NEXT:    retq
  store atomic <4 x bfloat> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec4_float_align(ptr %x, <4 x float> %v) nounwind {
; CHECK-SSE2-O3-LABEL: store_atomic_vec4_float_align:
; CHECK-SSE2-O3:       # %bb.0:
; CHECK-SSE2-O3-NEXT:    pushq %rax
; CHECK-SSE2-O3-NEXT:    movq %xmm0, %rsi
; CHECK-SSE2-O3-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-SSE2-O3-NEXT:    movq %xmm0, %rdx
; CHECK-SSE2-O3-NEXT:    movl $3, %ecx
; CHECK-SSE2-O3-NEXT:    callq __atomic_store_16@PLT
; CHECK-SSE2-O3-NEXT:    popq %rax
; CHECK-SSE2-O3-NEXT:    retq
;
; CHECK-SSE4-O3-LABEL: store_atomic_vec4_float_align:
; CHECK-SSE4-O3:       # %bb.0:
; CHECK-SSE4-O3-NEXT:    pushq %rbx
; CHECK-SSE4-O3-NEXT:    movdqa (%rdi), %xmm1
; CHECK-SSE4-O3-NEXT:    pextrq $1, %xmm0, %rcx
; CHECK-SSE4-O3-NEXT:    movq %xmm0, %rbx
; CHECK-SSE4-O3-NEXT:    .p2align 4
; CHECK-SSE4-O3-NEXT:  .LBB41_1: # %atomicrmw.start
; CHECK-SSE4-O3-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-SSE4-O3-NEXT:    movq %xmm1, %rax
; CHECK-SSE4-O3-NEXT:    pextrq $1, %xmm1, %rdx
; CHECK-SSE4-O3-NEXT:    lock cmpxchg16b (%rdi)
; CHECK-SSE4-O3-NEXT:    movq %rdx, %xmm0
; CHECK-SSE4-O3-NEXT:    movq %rax, %xmm1
; CHECK-SSE4-O3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
; CHECK-SSE4-O3-NEXT:    jne .LBB41_1
; CHECK-SSE4-O3-NEXT:  # %bb.2: # %atomicrmw.end
; CHECK-SSE4-O3-NEXT:    popq %rbx
; CHECK-SSE4-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_float_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovaps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec4_float_align:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    pushq %rax
; CHECK-SSE2-O0-NEXT:    movq %xmm0, %rsi
; CHECK-SSE2-O0-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-SSE2-O0-NEXT:    movq %xmm0, %rdx
; CHECK-SSE2-O0-NEXT:    movl $3, %ecx
; CHECK-SSE2-O0-NEXT:    callq __atomic_store_16@PLT
; CHECK-SSE2-O0-NEXT:    popq %rax
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec4_float_align:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    pushq %rbx
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-SSE4-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE4-O0-NEXT:    movaps (%rdi), %xmm0
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-SSE4-O0-NEXT:  .LBB41_1: # %atomicrmw.start
; CHECK-SSE4-O0-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-SSE4-O0-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-SSE4-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; CHECK-SSE4-O0-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; CHECK-SSE4-O0-NEXT:    pextrq $1, %xmm1, %rcx
; CHECK-SSE4-O0-NEXT:    movq %xmm1, %rbx
; CHECK-SSE4-O0-NEXT:    movq %xmm0, %rax
; CHECK-SSE4-O0-NEXT:    pextrq $1, %xmm0, %rdx
; CHECK-SSE4-O0-NEXT:    lock cmpxchg16b (%rsi)
; CHECK-SSE4-O0-NEXT:    movq %rax, %rcx
; CHECK-SSE4-O0-NEXT:    sete %al
; CHECK-SSE4-O0-NEXT:    movq %rdx, %xmm1
; CHECK-SSE4-O0-NEXT:    movq %rcx, %xmm0
; CHECK-SSE4-O0-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE4-O0-NEXT:    testb $1, %al
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-SSE4-O0-NEXT:    jne .LBB41_2
; CHECK-SSE4-O0-NEXT:    jmp .LBB41_1
; CHECK-SSE4-O0-NEXT:  .LBB41_2: # %atomicrmw.end
; CHECK-SSE4-O0-NEXT:    popq %rbx
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_float_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovaps %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <4 x float> %v, ptr %x release, align 16
  ret void
}

define <2 x half> @atomic_vec2_half(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_half:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_half:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_half:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_half:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x half>, ptr %x acquire, align 4
  ret <2 x half> %ret
}
define <2 x bfloat> @atomic_vec2_bfloat(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_bfloat:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_bfloat:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_bfloat:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_bfloat:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x bfloat>, ptr %x acquire, align 4
  ret <2 x bfloat> %ret
}
define <1 x ptr> @atomic_vec1_ptr(ptr %x) nounwind {
; CHECK-O3-LABEL: atomic_vec1_ptr:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    pushq %rax
; CHECK-O3-NEXT:    movq %rdi, %rsi
; CHECK-O3-NEXT:    movq %rsp, %rdx
; CHECK-O3-NEXT:    movl $8, %edi
; CHECK-O3-NEXT:    movl $2, %ecx
; CHECK-O3-NEXT:    callq __atomic_load@PLT
; CHECK-O3-NEXT:    movq (%rsp), %rax
; CHECK-O3-NEXT:    popq %rcx
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: atomic_vec1_ptr:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    pushq %rax
; CHECK-O0-NEXT:    movq %rdi, %rsi
; CHECK-O0-NEXT:    movl $8, %edi
; CHECK-O0-NEXT:    movq %rsp, %rdx
; CHECK-O0-NEXT:    movl $2, %ecx
; CHECK-O0-NEXT:    callq __atomic_load@PLT
; CHECK-O0-NEXT:    movq (%rsp), %rax
; CHECK-O0-NEXT:    popq %rcx
; CHECK-O0-NEXT:    retq
  %ret = load atomic <1 x ptr>, ptr %x acquire, align 4
  ret <1 x ptr> %ret
}

define <1 x half> @atomic_vec1_half(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec1_half:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-SSE-O3-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_half:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-AVX-O3-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_half:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movw (%rdi), %cx
; CHECK-SSE-O0-NEXT:    # implicit-def: $eax
; CHECK-SSE-O0-NEXT:    movw %cx, %ax
; CHECK-SSE-O0-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_half:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    movw (%rdi), %cx
; CHECK-AVX-O0-NEXT:    # implicit-def: $eax
; CHECK-AVX-O0-NEXT:    movw %cx, %ax
; CHECK-AVX-O0-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <1 x half>, ptr %x acquire, align 2
  ret <1 x half> %ret
}

define <1 x float> @atomic_vec1_float(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec1_float:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_float:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_float:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_float:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <1 x float>, ptr %x acquire, align 4
  ret <1 x float> %ret
}

define <1 x double> @atomic_vec1_double_align(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec1_double_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_double_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_double_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_double_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <1 x double>, ptr %x acquire, align 8
  ret <1 x double> %ret
}

define <1 x i64> @atomic_vec1_i64(ptr %x) nounwind {
; CHECK-O3-LABEL: atomic_vec1_i64:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    pushq %rax
; CHECK-O3-NEXT:    movq %rdi, %rsi
; CHECK-O3-NEXT:    movq %rsp, %rdx
; CHECK-O3-NEXT:    movl $8, %edi
; CHECK-O3-NEXT:    movl $2, %ecx
; CHECK-O3-NEXT:    callq __atomic_load@PLT
; CHECK-O3-NEXT:    movq (%rsp), %rax
; CHECK-O3-NEXT:    popq %rcx
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: atomic_vec1_i64:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    pushq %rax
; CHECK-O0-NEXT:    movq %rdi, %rsi
; CHECK-O0-NEXT:    movl $8, %edi
; CHECK-O0-NEXT:    movq %rsp, %rdx
; CHECK-O0-NEXT:    movl $2, %ecx
; CHECK-O0-NEXT:    callq __atomic_load@PLT
; CHECK-O0-NEXT:    movq (%rsp), %rax
; CHECK-O0-NEXT:    popq %rcx
; CHECK-O0-NEXT:    retq
  %ret = load atomic <1 x i64>, ptr %x acquire, align 4
  ret <1 x i64> %ret
}

define <1 x double> @atomic_vec1_double(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec1_double:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    pushq %rax
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $8, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O3-NEXT:    popq %rax
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_double:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    pushq %rax
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $8, %edi
; CHECK-AVX-O3-NEXT:    movl $2, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O3-NEXT:    popq %rax
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_double:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    pushq %rax
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $8, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O0-NEXT:    popq %rax
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_double:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    pushq %rax
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    movl $8, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $2, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O0-NEXT:    popq %rax
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <1 x double>, ptr %x acquire, align 4
  ret <1 x double> %ret
}

define <2 x i32> @atomic_vec2_i32(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec2_i32:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    pushq %rax
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $8, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O3-NEXT:    popq %rax
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_i32:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    pushq %rax
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $8, %edi
; CHECK-AVX-O3-NEXT:    movl $2, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O3-NEXT:    popq %rax
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_i32:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    pushq %rax
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $8, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O0-NEXT:    popq %rax
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_i32:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    pushq %rax
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    movl $8, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $2, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O0-NEXT:    popq %rax
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x i32>, ptr %x acquire, align 4
  ret <2 x i32> %ret
}

define <4 x i8> @atomic_vec4_i8(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_i8:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_i8:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_i8:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_i8:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <4 x i8>, ptr %x acquire, align 4
  ret <4 x i8> %ret
}

define <4 x i16> @atomic_vec4_i16(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_i16:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_i16:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_i16:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_i16:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <4 x i16>, ptr %x acquire, align 8
  ret <4 x i16> %ret
}

define <4 x half> @atomic_vec4_half(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_half:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_half:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_half:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_half:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <4 x half>, ptr %x acquire, align 8
  ret <4 x half> %ret
}
define <4 x bfloat> @atomic_vec4_bfloat(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_bfloat:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_bfloat:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_bfloat:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_bfloat:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <4 x bfloat>, ptr %x acquire, align 8
  ret <4 x bfloat> %ret
}

define <4 x float> @atomic_vec4_float(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_float:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $24, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $16, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT:    addq $24, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_float:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    subq $24, %rsp
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $16, %edi
; CHECK-AVX-O3-NEXT:    movl $2, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT:    vmovaps (%rsp), %xmm0
; CHECK-AVX-O3-NEXT:    addq $24, %rsp
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_float:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $24, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $16, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT:    addq $24, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_float:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    subq $24, %rsp
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    movl $16, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $2, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT:    vmovaps (%rsp), %xmm0
; CHECK-AVX-O0-NEXT:    addq $24, %rsp
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <4 x float>, ptr %x acquire, align 4
  ret <4 x float> %ret
}

define <4 x float> @atomic_vec4_float_align(ptr %x) nounwind {
; CHECK-SSE2-O3-LABEL: atomic_vec4_float_align:
; CHECK-SSE2-O3:       # %bb.0:
; CHECK-SSE2-O3-NEXT:    pushq %rax
; CHECK-SSE2-O3-NEXT:    movl $2, %esi
; CHECK-SSE2-O3-NEXT:    callq __atomic_load_16@PLT
; CHECK-SSE2-O3-NEXT:    movq %rdx, %xmm1
; CHECK-SSE2-O3-NEXT:    movq %rax, %xmm0
; CHECK-SSE2-O3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O3-NEXT:    popq %rax
; CHECK-SSE2-O3-NEXT:    retq
;
; CHECK-SSE4-O3-LABEL: atomic_vec4_float_align:
; CHECK-SSE4-O3:       # %bb.0:
; CHECK-SSE4-O3-NEXT:    pushq %rbx
; CHECK-SSE4-O3-NEXT:    xorl %eax, %eax
; CHECK-SSE4-O3-NEXT:    xorl %edx, %edx
; CHECK-SSE4-O3-NEXT:    xorl %ecx, %ecx
; CHECK-SSE4-O3-NEXT:    xorl %ebx, %ebx
; CHECK-SSE4-O3-NEXT:    lock cmpxchg16b (%rdi)
; CHECK-SSE4-O3-NEXT:    movq %rdx, %xmm1
; CHECK-SSE4-O3-NEXT:    movq %rax, %xmm0
; CHECK-SSE4-O3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE4-O3-NEXT:    popq %rbx
; CHECK-SSE4-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_float_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovaps (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: atomic_vec4_float_align:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    pushq %rax
; CHECK-SSE2-O0-NEXT:    movl $2, %esi
; CHECK-SSE2-O0-NEXT:    callq __atomic_load_16@PLT
; CHECK-SSE2-O0-NEXT:    movq %rdx, %xmm1
; CHECK-SSE2-O0-NEXT:    movq %rax, %xmm0
; CHECK-SSE2-O0-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O0-NEXT:    popq %rax
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: atomic_vec4_float_align:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    pushq %rbx
; CHECK-SSE4-O0-NEXT:    xorl %eax, %eax
; CHECK-SSE4-O0-NEXT:    movl %eax, %ebx
; CHECK-SSE4-O0-NEXT:    movq %rbx, %rax
; CHECK-SSE4-O0-NEXT:    movq %rbx, %rdx
; CHECK-SSE4-O0-NEXT:    movq %rbx, %rcx
; CHECK-SSE4-O0-NEXT:    lock cmpxchg16b (%rdi)
; CHECK-SSE4-O0-NEXT:    movq %rdx, %xmm1
; CHECK-SSE4-O0-NEXT:    movq %rax, %xmm0
; CHECK-SSE4-O0-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE4-O0-NEXT:    popq %rbx
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_float_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovaps (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <4 x float>, ptr %x acquire, align 16
  ret <4 x float> %ret
}

define <8 x double> @atomic_vec8_double(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec8_double:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $72, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $64, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O3-NEXT:    addq $72, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX2-O3-LABEL: atomic_vec8_double:
; CHECK-AVX2-O3:       # %bb.0:
; CHECK-AVX2-O3-NEXT:    subq $72, %rsp
; CHECK-AVX2-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT:    movl $64, %edi
; CHECK-AVX2-O3-NEXT:    movl $2, %ecx
; CHECK-AVX2-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX2-O3-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX2-O3-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O3-NEXT:    addq $72, %rsp
; CHECK-AVX2-O3-NEXT:    retq
;
; CHECK-AVX512-O3-LABEL: atomic_vec8_double:
; CHECK-AVX512-O3:       # %bb.0:
; CHECK-AVX512-O3-NEXT:    subq $72, %rsp
; CHECK-AVX512-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT:    movl $64, %edi
; CHECK-AVX512-O3-NEXT:    movl $2, %ecx
; CHECK-AVX512-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX512-O3-NEXT:    vmovups (%rsp), %zmm0
; CHECK-AVX512-O3-NEXT:    addq $72, %rsp
; CHECK-AVX512-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec8_double:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $72, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $64, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movapd (%rsp), %xmm0
; CHECK-SSE-O0-NEXT:    movapd {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT:    movapd {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O0-NEXT:    movapd {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O0-NEXT:    addq $72, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: atomic_vec8_double:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $72, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT:    movl $64, %edi
; CHECK-AVX2-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT:    movl $2, %ecx
; CHECK-AVX2-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX2-O0-NEXT:    vmovupd (%rsp), %ymm0
; CHECK-AVX2-O0-NEXT:    vmovupd {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O0-NEXT:    addq $72, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: atomic_vec8_double:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $72, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    movl $64, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $2, %ecx
; CHECK-AVX512-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX512-O0-NEXT:    vmovupd (%rsp), %zmm0
; CHECK-AVX512-O0-NEXT:    addq $72, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  %ret = load atomic <8 x double>, ptr %x acquire, align 4
  ret <8 x double> %ret
}

define <16 x bfloat> @atomic_vec16_bfloat(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec16_bfloat:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $40, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $32, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT:    addq $40, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec16_bfloat:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    subq $40, %rsp
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $32, %edi
; CHECK-AVX-O3-NEXT:    movl $2, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX-O3-NEXT:    addq $40, %rsp
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec16_bfloat:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $40, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $32, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT:    addq $40, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec16_bfloat:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    subq $40, %rsp
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    movl $32, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $2, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX-O0-NEXT:    addq $40, %rsp
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <16 x bfloat>, ptr %x acquire, align 4
  ret <16 x bfloat> %ret
}

define <32 x half> @atomic_vec32_half(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec32_half:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $72, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $64, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O3-NEXT:    addq $72, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX2-O3-LABEL: atomic_vec32_half:
; CHECK-AVX2-O3:       # %bb.0:
; CHECK-AVX2-O3-NEXT:    subq $72, %rsp
; CHECK-AVX2-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT:    movl $64, %edi
; CHECK-AVX2-O3-NEXT:    movl $2, %ecx
; CHECK-AVX2-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX2-O3-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX2-O3-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O3-NEXT:    addq $72, %rsp
; CHECK-AVX2-O3-NEXT:    retq
;
; CHECK-AVX512-O3-LABEL: atomic_vec32_half:
; CHECK-AVX512-O3:       # %bb.0:
; CHECK-AVX512-O3-NEXT:    subq $72, %rsp
; CHECK-AVX512-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT:    movl $64, %edi
; CHECK-AVX512-O3-NEXT:    movl $2, %ecx
; CHECK-AVX512-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX512-O3-NEXT:    vmovups (%rsp), %zmm0
; CHECK-AVX512-O3-NEXT:    addq $72, %rsp
; CHECK-AVX512-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec32_half:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $72, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $64, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O0-NEXT:    addq $72, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: atomic_vec32_half:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $72, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT:    movl $64, %edi
; CHECK-AVX2-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT:    movl $2, %ecx
; CHECK-AVX2-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX2-O0-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX2-O0-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O0-NEXT:    addq $72, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: atomic_vec32_half:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $72, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    movl $64, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $2, %ecx
; CHECK-AVX512-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX512-O0-NEXT:    vmovups (%rsp), %zmm0
; CHECK-AVX512-O0-NEXT:    addq $72, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  %ret = load atomic <32 x half>, ptr %x acquire, align 4
  ret <32 x half> %ret
}

define <8 x float> @atomic_vec8_float_align(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec8_float_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $40, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $32, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT:    addq $40, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec8_float_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    subq $40, %rsp
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $32, %edi
; CHECK-AVX-O3-NEXT:    movl $2, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX-O3-NEXT:    addq $40, %rsp
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec8_float_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $40, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $32, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT:    addq $40, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec8_float_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    subq $40, %rsp
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    movl $32, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $2, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX-O0-NEXT:    addq $40, %rsp
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <8 x float>, ptr %x acquire, align 32
  ret <8 x float> %ret
}

define <16 x float> @atomic_vec16_float_align(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec16_float_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $72, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $64, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O3-NEXT:    addq $72, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX2-O3-LABEL: atomic_vec16_float_align:
; CHECK-AVX2-O3:       # %bb.0:
; CHECK-AVX2-O3-NEXT:    subq $72, %rsp
; CHECK-AVX2-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT:    movl $64, %edi
; CHECK-AVX2-O3-NEXT:    movl $2, %ecx
; CHECK-AVX2-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX2-O3-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX2-O3-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O3-NEXT:    addq $72, %rsp
; CHECK-AVX2-O3-NEXT:    retq
;
; CHECK-AVX512-O3-LABEL: atomic_vec16_float_align:
; CHECK-AVX512-O3:       # %bb.0:
; CHECK-AVX512-O3-NEXT:    subq $72, %rsp
; CHECK-AVX512-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT:    movl $64, %edi
; CHECK-AVX512-O3-NEXT:    movl $2, %ecx
; CHECK-AVX512-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX512-O3-NEXT:    vmovups (%rsp), %zmm0
; CHECK-AVX512-O3-NEXT:    addq $72, %rsp
; CHECK-AVX512-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec16_float_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $72, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $64, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O0-NEXT:    addq $72, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: atomic_vec16_float_align:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $72, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT:    movl $64, %edi
; CHECK-AVX2-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT:    movl $2, %ecx
; CHECK-AVX2-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX2-O0-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX2-O0-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O0-NEXT:    addq $72, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: atomic_vec16_float_align:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $72, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    movl $64, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $2, %ecx
; CHECK-AVX512-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX512-O0-NEXT:    vmovups (%rsp), %zmm0
; CHECK-AVX512-O0-NEXT:    addq $72, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  %ret = load atomic <16 x float>, ptr %x acquire, align 64
  ret <16 x float> %ret
}

define <8 x double> @atomic_vec8_double_align(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec8_double_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $72, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $64, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O3-NEXT:    addq $72, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX2-O3-LABEL: atomic_vec8_double_align:
; CHECK-AVX2-O3:       # %bb.0:
; CHECK-AVX2-O3-NEXT:    subq $72, %rsp
; CHECK-AVX2-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT:    movl $64, %edi
; CHECK-AVX2-O3-NEXT:    movl $2, %ecx
; CHECK-AVX2-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX2-O3-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX2-O3-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O3-NEXT:    addq $72, %rsp
; CHECK-AVX2-O3-NEXT:    retq
;
; CHECK-AVX512-O3-LABEL: atomic_vec8_double_align:
; CHECK-AVX512-O3:       # %bb.0:
; CHECK-AVX512-O3-NEXT:    subq $72, %rsp
; CHECK-AVX512-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT:    movl $64, %edi
; CHECK-AVX512-O3-NEXT:    movl $2, %ecx
; CHECK-AVX512-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX512-O3-NEXT:    vmovups (%rsp), %zmm0
; CHECK-AVX512-O3-NEXT:    addq $72, %rsp
; CHECK-AVX512-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec8_double_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $72, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $64, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movapd (%rsp), %xmm0
; CHECK-SSE-O0-NEXT:    movapd {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT:    movapd {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O0-NEXT:    movapd {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O0-NEXT:    addq $72, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: atomic_vec8_double_align:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $72, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT:    movl $64, %edi
; CHECK-AVX2-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT:    movl $2, %ecx
; CHECK-AVX2-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX2-O0-NEXT:    vmovupd (%rsp), %ymm0
; CHECK-AVX2-O0-NEXT:    vmovupd {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O0-NEXT:    addq $72, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: atomic_vec8_double_align:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $72, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    movl $64, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $2, %ecx
; CHECK-AVX512-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX512-O0-NEXT:    vmovupd (%rsp), %zmm0
; CHECK-AVX512-O0-NEXT:    addq $72, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  %ret = load atomic <8 x double>, ptr %x acquire, align 64
  ret <8 x double> %ret
}

define <16 x bfloat> @atomic_vec16_bfloat_align(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec16_bfloat_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $40, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $32, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT:    addq $40, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec16_bfloat_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    subq $40, %rsp
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $32, %edi
; CHECK-AVX-O3-NEXT:    movl $2, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX-O3-NEXT:    addq $40, %rsp
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec16_bfloat_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $40, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $32, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT:    addq $40, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec16_bfloat_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    subq $40, %rsp
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    movl $32, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $2, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX-O0-NEXT:    addq $40, %rsp
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <16 x bfloat>, ptr %x acquire, align 32
  ret <16 x bfloat> %ret
}

define <32 x half> @atomic_vec32_half_align(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec32_half_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $72, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $64, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O3-NEXT:    addq $72, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX2-O3-LABEL: atomic_vec32_half_align:
; CHECK-AVX2-O3:       # %bb.0:
; CHECK-AVX2-O3-NEXT:    subq $72, %rsp
; CHECK-AVX2-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT:    movl $64, %edi
; CHECK-AVX2-O3-NEXT:    movl $2, %ecx
; CHECK-AVX2-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX2-O3-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX2-O3-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O3-NEXT:    addq $72, %rsp
; CHECK-AVX2-O3-NEXT:    retq
;
; CHECK-AVX512-O3-LABEL: atomic_vec32_half_align:
; CHECK-AVX512-O3:       # %bb.0:
; CHECK-AVX512-O3-NEXT:    subq $72, %rsp
; CHECK-AVX512-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT:    movl $64, %edi
; CHECK-AVX512-O3-NEXT:    movl $2, %ecx
; CHECK-AVX512-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX512-O3-NEXT:    vmovups (%rsp), %zmm0
; CHECK-AVX512-O3-NEXT:    addq $72, %rsp
; CHECK-AVX512-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec32_half_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $72, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $64, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O0-NEXT:    addq $72, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: atomic_vec32_half_align:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $72, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT:    movl $64, %edi
; CHECK-AVX2-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT:    movl $2, %ecx
; CHECK-AVX2-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX2-O0-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX2-O0-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O0-NEXT:    addq $72, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: atomic_vec32_half_align:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $72, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    movl $64, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $2, %ecx
; CHECK-AVX512-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX512-O0-NEXT:    vmovups (%rsp), %zmm0
; CHECK-AVX512-O0-NEXT:    addq $72, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  %ret = load atomic <32 x half>, ptr %x acquire, align 64
  ret <32 x half> %ret
}

define void @store_atomic_vec1_ptr_unaligned(ptr %x, <1 x ptr> %v) nounwind {
; CHECK-O3-LABEL: store_atomic_vec1_ptr_unaligned:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    pushq %rax
; CHECK-O3-NEXT:    movq %rdi, %rax
; CHECK-O3-NEXT:    movq %rsi, (%rsp)
; CHECK-O3-NEXT:    movq %rsp, %rdx
; CHECK-O3-NEXT:    movl $8, %edi
; CHECK-O3-NEXT:    movq %rax, %rsi
; CHECK-O3-NEXT:    movl $3, %ecx
; CHECK-O3-NEXT:    callq __atomic_store@PLT
; CHECK-O3-NEXT:    popq %rax
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: store_atomic_vec1_ptr_unaligned:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    pushq %rax
; CHECK-O0-NEXT:    movq %rsi, %rax
; CHECK-O0-NEXT:    movq %rdi, %rsi
; CHECK-O0-NEXT:    movq %rax, (%rsp)
; CHECK-O0-NEXT:    movl $8, %edi
; CHECK-O0-NEXT:    movq %rsp, %rdx
; CHECK-O0-NEXT:    movl $3, %ecx
; CHECK-O0-NEXT:    callq __atomic_store@PLT
; CHECK-O0-NEXT:    popq %rax
; CHECK-O0-NEXT:    retq
  store atomic <1 x ptr> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec1_i64_unaligned(ptr %x, <1 x i64> %v) nounwind {
; CHECK-O3-LABEL: store_atomic_vec1_i64_unaligned:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    pushq %rax
; CHECK-O3-NEXT:    movq %rdi, %rax
; CHECK-O3-NEXT:    movq %rsi, (%rsp)
; CHECK-O3-NEXT:    movq %rsp, %rdx
; CHECK-O3-NEXT:    movl $8, %edi
; CHECK-O3-NEXT:    movq %rax, %rsi
; CHECK-O3-NEXT:    movl $3, %ecx
; CHECK-O3-NEXT:    callq __atomic_store@PLT
; CHECK-O3-NEXT:    popq %rax
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: store_atomic_vec1_i64_unaligned:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    pushq %rax
; CHECK-O0-NEXT:    movq %rsi, %rax
; CHECK-O0-NEXT:    movq %rdi, %rsi
; CHECK-O0-NEXT:    movq %rax, (%rsp)
; CHECK-O0-NEXT:    movl $8, %edi
; CHECK-O0-NEXT:    movq %rsp, %rdx
; CHECK-O0-NEXT:    movl $3, %ecx
; CHECK-O0-NEXT:    callq __atomic_store@PLT
; CHECK-O0-NEXT:    popq %rax
; CHECK-O0-NEXT:    retq
  store atomic <1 x i64> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec1_double_unaligned(ptr %x, <1 x double> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_double_unaligned:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    pushq %rax
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movsd %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $8, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    popq %rax
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_double_unaligned:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    pushq %rax
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    vmovsd %xmm0, (%rsp)
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $8, %edi
; CHECK-AVX-O3-NEXT:    movl $3, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT:    popq %rax
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_double_unaligned:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    pushq %rax
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movsd %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT:    movl $8, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $3, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT:    popq %rax
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_double_unaligned:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    pushq %rax
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    vmovsd %xmm0, (%rsp)
; CHECK-AVX-O0-NEXT:    movl $8, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $3, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O0-NEXT:    popq %rax
; CHECK-AVX-O0-NEXT:    retq
  store atomic <1 x double> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec2_i32_unaligned(ptr %x, <2 x i32> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_i32_unaligned:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    pushq %rax
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movlps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $8, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    popq %rax
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i32_unaligned:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    pushq %rax
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    vmovlps %xmm0, (%rsp)
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $8, %edi
; CHECK-AVX-O3-NEXT:    movl $3, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT:    popq %rax
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_i32_unaligned:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    pushq %rax
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movq %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT:    movl $8, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $3, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT:    popq %rax
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i32_unaligned:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    pushq %rax
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    vmovq %xmm0, (%rsp)
; CHECK-AVX-O0-NEXT:    movl $8, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $3, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O0-NEXT:    popq %rax
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x i32> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec4_float_unaligned(ptr %x, <4 x float> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_float_unaligned:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $24, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $16, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    addq $24, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_float_unaligned:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    subq $24, %rsp
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    vmovaps %xmm0, (%rsp)
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $16, %edi
; CHECK-AVX-O3-NEXT:    movl $3, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT:    addq $24, %rsp
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec4_float_unaligned:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $24, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT:    movl $16, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $3, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT:    addq $24, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_float_unaligned:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    subq $24, %rsp
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    vmovaps %xmm0, (%rsp)
; CHECK-AVX-O0-NEXT:    movl $16, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $3, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O0-NEXT:    addq $24, %rsp
; CHECK-AVX-O0-NEXT:    retq
  store atomic <4 x float> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec8_double_unaligned(ptr %x, <8 x double> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $72, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $64, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    addq $72, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX2-O3-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-AVX2-O3:       # %bb.0:
; CHECK-AVX2-O3-NEXT:    subq $72, %rsp
; CHECK-AVX2-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O3-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX2-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT:    movl $64, %edi
; CHECK-AVX2-O3-NEXT:    movl $3, %ecx
; CHECK-AVX2-O3-NEXT:    vzeroupper
; CHECK-AVX2-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O3-NEXT:    addq $72, %rsp
; CHECK-AVX2-O3-NEXT:    retq
;
; CHECK-AVX512-O3-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-AVX512-O3:       # %bb.0:
; CHECK-AVX512-O3-NEXT:    subq $72, %rsp
; CHECK-AVX512-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT:    vmovups %zmm0, (%rsp)
; CHECK-AVX512-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT:    movl $64, %edi
; CHECK-AVX512-O3-NEXT:    movl $3, %ecx
; CHECK-AVX512-O3-NEXT:    vzeroupper
; CHECK-AVX512-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O3-NEXT:    addq $72, %rsp
; CHECK-AVX512-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $72, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movapd %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT:    movapd %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movapd %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movapd %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movl $64, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $3, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT:    addq $72, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $72, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT:    vmovupd %ymm0, (%rsp)
; CHECK-AVX2-O0-NEXT:    vmovupd %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    movl $64, %edi
; CHECK-AVX2-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT:    movl $3, %ecx
; CHECK-AVX2-O0-NEXT:    vzeroupper
; CHECK-AVX2-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT:    addq $72, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $72, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    vmovupd %zmm0, (%rsp)
; CHECK-AVX512-O0-NEXT:    movl $64, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $3, %ecx
; CHECK-AVX512-O0-NEXT:    vzeroupper
; CHECK-AVX512-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT:    addq $72, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  store atomic <8 x double> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec16_bfloat_unaligned(ptr %x, <16 x bfloat> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $40, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $32, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    addq $40, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    subq $40, %rsp
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $32, %edi
; CHECK-AVX-O3-NEXT:    movl $3, %ecx
; CHECK-AVX-O3-NEXT:    vzeroupper
; CHECK-AVX-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT:    addq $40, %rsp
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    subq $120, %rsp
; CHECK-SSE2-O0-NEXT:    movaps %xmm1, %xmm2
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE2-O0-NEXT:    xorps %xmm13, %xmm13
; CHECK-SSE2-O0-NEXT:    movaps %xmm13, %xmm15
; CHECK-SSE2-O0-NEXT:    punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
; CHECK-SSE2-O0-NEXT:    punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]
; CHECK-SSE2-O0-NEXT:    movaps %xmm2, %xmm14
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm14
; CHECK-SSE2-O0-NEXT:    movaps %xmm1, %xmm0
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm0
; CHECK-SSE2-O0-NEXT:    pextrw $7, %xmm2, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm12
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm12
; CHECK-SSE2-O0-NEXT:    pextrw $6, %xmm2, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm11
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm11
; CHECK-SSE2-O0-NEXT:    pextrw $5, %xmm2, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm10
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm10
; CHECK-SSE2-O0-NEXT:    pextrw $3, %xmm2, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm9
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm9
; CHECK-SSE2-O0-NEXT:    pextrw $2, %xmm2, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm8
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm8
; CHECK-SSE2-O0-NEXT:    pextrw $1, %xmm2, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm7
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm7
; CHECK-SSE2-O0-NEXT:    pextrw $7, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm6
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm6
; CHECK-SSE2-O0-NEXT:    pextrw $6, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm5
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm5
; CHECK-SSE2-O0-NEXT:    pextrw $5, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm4
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm4
; CHECK-SSE2-O0-NEXT:    pextrw $3, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm3
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm3
; CHECK-SSE2-O0-NEXT:    pextrw $2, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm2
; CHECK-SSE2-O0-NEXT:    pextrw $1, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm1
; CHECK-SSE2-O0-NEXT:    movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    movl $32, %edi
; CHECK-SSE2-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
; CHECK-SSE2-O0-NEXT:    movl $3, %ecx
; CHECK-SSE2-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE2-O0-NEXT:    addq $120, %rsp
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    subq $120, %rsp
; CHECK-SSE4-O0-NEXT:    movaps %xmm1, %xmm2
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE4-O0-NEXT:    xorps %xmm0, %xmm0
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm15
; CHECK-SSE4-O0-NEXT:    punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
; CHECK-SSE4-O0-NEXT:    movaps %xmm2, %xmm14
; CHECK-SSE4-O0-NEXT:    pblendw {{.*#+}} xmm14 = xmm0[0],xmm14[1],xmm0[2],xmm14[3],xmm0[4],xmm14[5],xmm0[6],xmm14[7]
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm12
; CHECK-SSE4-O0-NEXT:    punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm1[4],xmm12[5],xmm1[5],xmm12[6],xmm1[6],xmm12[7],xmm1[7]
; CHECK-SSE4-O0-NEXT:    movaps %xmm1, %xmm11
; CHECK-SSE4-O0-NEXT:    pblendw {{.*#+}} xmm11 = xmm0[0],xmm11[1],xmm0[2],xmm11[3],xmm0[4],xmm11[5],xmm0[6],xmm11[7]
; CHECK-SSE4-O0-NEXT:    movaps %xmm2, %xmm13
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm13
; CHECK-SSE4-O0-NEXT:    movaps %xmm1, %xmm0
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm0
; CHECK-SSE4-O0-NEXT:    pextrw $7, %xmm2, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm10
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm10
; CHECK-SSE4-O0-NEXT:    pextrw $6, %xmm2, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm9
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm9
; CHECK-SSE4-O0-NEXT:    pextrw $5, %xmm2, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm8
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm8
; CHECK-SSE4-O0-NEXT:    pextrw $3, %xmm2, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm7
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm7
; CHECK-SSE4-O0-NEXT:    pextrw $2, %xmm2, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm6
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm6
; CHECK-SSE4-O0-NEXT:    pextrw $7, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm5
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm5
; CHECK-SSE4-O0-NEXT:    pextrw $6, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm4
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm4
; CHECK-SSE4-O0-NEXT:    pextrw $5, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm3
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm3
; CHECK-SSE4-O0-NEXT:    pextrw $3, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm2
; CHECK-SSE4-O0-NEXT:    pextrw $2, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm1
; CHECK-SSE4-O0-NEXT:    movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    movl $32, %edi
; CHECK-SSE4-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
; CHECK-SSE4-O0-NEXT:    movl $3, %ecx
; CHECK-SSE4-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE4-O0-NEXT:    addq $120, %rsp
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $120, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-AVX2-O0-NEXT:    vextracti128 $1, %ymm0, %xmm8
; CHECK-AVX2-O0-NEXT:    vmovdqa {{.*#+}} xmm6 = [128,128,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm6, %xmm8, %xmm15
; CHECK-AVX2-O0-NEXT:    vmovdqa {{.*#+}} xmm5 = [128,128,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm5, %xmm8, %xmm14
; CHECK-AVX2-O0-NEXT:    vmovdqa {{.*#+}} xmm4 = [128,128,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm4, %xmm8, %xmm13
; CHECK-AVX2-O0-NEXT:    vxorps %xmm1, %xmm1, %xmm1
; CHECK-AVX2-O0-NEXT:    vpunpckhwd {{.*#+}} xmm12 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]
; CHECK-AVX2-O0-NEXT:    vmovdqa {{.*#+}} xmm3 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm3, %xmm8, %xmm11
; CHECK-AVX2-O0-NEXT:    vmovdqa {{.*#+}} xmm2 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm2, %xmm8, %xmm10
; CHECK-AVX2-O0-NEXT:    vpblendw {{.*#+}} xmm9 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
; CHECK-AVX2-O0-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm6, %xmm0, %xmm7
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm5, %xmm0, %xmm6
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm4, %xmm0, %xmm5
; CHECK-AVX2-O0-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm3, %xmm0, %xmm3
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm2, %xmm0, %xmm2
; CHECK-AVX2-O0-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; CHECK-AVX2-O0-NEXT:    vpslld $16, %xmm8, %xmm8
; CHECK-AVX2-O0-NEXT:    vpslld $16, %xmm0, %xmm0
; CHECK-AVX2-O0-NEXT:    vmovss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vzeroupper
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    movl $32, %edi
; CHECK-AVX2-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
; CHECK-AVX2-O0-NEXT:    movl $3, %ecx
; CHECK-AVX2-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT:    addq $120, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $40, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX512-O0-NEXT:    movl $32, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $3, %ecx
; CHECK-AVX512-O0-NEXT:    vzeroupper
; CHECK-AVX512-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT:    addq $40, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  store atomic <16 x bfloat> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec32_half_unaligned(ptr %x, <32 x half> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $72, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $64, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    addq $72, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX2-O3-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-AVX2-O3:       # %bb.0:
; CHECK-AVX2-O3-NEXT:    subq $72, %rsp
; CHECK-AVX2-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O3-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX2-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT:    movl $64, %edi
; CHECK-AVX2-O3-NEXT:    movl $3, %ecx
; CHECK-AVX2-O3-NEXT:    vzeroupper
; CHECK-AVX2-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O3-NEXT:    addq $72, %rsp
; CHECK-AVX2-O3-NEXT:    retq
;
; CHECK-AVX512-O3-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-AVX512-O3:       # %bb.0:
; CHECK-AVX512-O3-NEXT:    subq $72, %rsp
; CHECK-AVX512-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT:    vmovups %zmm0, (%rsp)
; CHECK-AVX512-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT:    movl $64, %edi
; CHECK-AVX512-O3-NEXT:    movl $3, %ecx
; CHECK-AVX512-O3-NEXT:    vzeroupper
; CHECK-AVX512-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O3-NEXT:    addq $72, %rsp
; CHECK-AVX512-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $72, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movl $64, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $3, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT:    addq $72, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $72, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX2-O0-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    movl $64, %edi
; CHECK-AVX2-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT:    movl $3, %ecx
; CHECK-AVX2-O0-NEXT:    vzeroupper
; CHECK-AVX2-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT:    addq $72, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $72, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    vmovups %zmm0, (%rsp)
; CHECK-AVX512-O0-NEXT:    movl $64, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $3, %ecx
; CHECK-AVX512-O0-NEXT:    vzeroupper
; CHECK-AVX512-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT:    addq $72, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  store atomic <32 x half> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec8_float_align(ptr %x, <8 x float> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec8_float_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $40, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $32, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    addq $40, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec8_float_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    subq $40, %rsp
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $32, %edi
; CHECK-AVX-O3-NEXT:    movl $3, %ecx
; CHECK-AVX-O3-NEXT:    vzeroupper
; CHECK-AVX-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT:    addq $40, %rsp
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec8_float_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $40, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movl $32, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $3, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT:    addq $40, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec8_float_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    subq $40, %rsp
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX-O0-NEXT:    movl $32, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $3, %ecx
; CHECK-AVX-O0-NEXT:    vzeroupper
; CHECK-AVX-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O0-NEXT:    addq $40, %rsp
; CHECK-AVX-O0-NEXT:    retq
  store atomic <8 x float> %v, ptr %x release, align 32
  ret void
}

define void @store_atomic_vec16_float_align(ptr %x, <16 x float> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec16_float_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $72, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $64, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    addq $72, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX2-O3-LABEL: store_atomic_vec16_float_align:
; CHECK-AVX2-O3:       # %bb.0:
; CHECK-AVX2-O3-NEXT:    subq $72, %rsp
; CHECK-AVX2-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O3-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX2-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT:    movl $64, %edi
; CHECK-AVX2-O3-NEXT:    movl $3, %ecx
; CHECK-AVX2-O3-NEXT:    vzeroupper
; CHECK-AVX2-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O3-NEXT:    addq $72, %rsp
; CHECK-AVX2-O3-NEXT:    retq
;
; CHECK-AVX512-O3-LABEL: store_atomic_vec16_float_align:
; CHECK-AVX512-O3:       # %bb.0:
; CHECK-AVX512-O3-NEXT:    subq $72, %rsp
; CHECK-AVX512-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT:    vmovups %zmm0, (%rsp)
; CHECK-AVX512-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT:    movl $64, %edi
; CHECK-AVX512-O3-NEXT:    movl $3, %ecx
; CHECK-AVX512-O3-NEXT:    vzeroupper
; CHECK-AVX512-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O3-NEXT:    addq $72, %rsp
; CHECK-AVX512-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec16_float_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $72, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movl $64, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $3, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT:    addq $72, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec16_float_align:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $72, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX2-O0-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    movl $64, %edi
; CHECK-AVX2-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT:    movl $3, %ecx
; CHECK-AVX2-O0-NEXT:    vzeroupper
; CHECK-AVX2-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT:    addq $72, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec16_float_align:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $72, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    vmovups %zmm0, (%rsp)
; CHECK-AVX512-O0-NEXT:    movl $64, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $3, %ecx
; CHECK-AVX512-O0-NEXT:    vzeroupper
; CHECK-AVX512-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT:    addq $72, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  store atomic <16 x float> %v, ptr %x release, align 64
  ret void
}

define void @store_atomic_vec8_double_align(ptr %x, <8 x double> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec8_double_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $72, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $64, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    addq $72, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX2-O3-LABEL: store_atomic_vec8_double_align:
; CHECK-AVX2-O3:       # %bb.0:
; CHECK-AVX2-O3-NEXT:    subq $72, %rsp
; CHECK-AVX2-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O3-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX2-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT:    movl $64, %edi
; CHECK-AVX2-O3-NEXT:    movl $3, %ecx
; CHECK-AVX2-O3-NEXT:    vzeroupper
; CHECK-AVX2-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O3-NEXT:    addq $72, %rsp
; CHECK-AVX2-O3-NEXT:    retq
;
; CHECK-AVX512-O3-LABEL: store_atomic_vec8_double_align:
; CHECK-AVX512-O3:       # %bb.0:
; CHECK-AVX512-O3-NEXT:    subq $72, %rsp
; CHECK-AVX512-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT:    vmovups %zmm0, (%rsp)
; CHECK-AVX512-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT:    movl $64, %edi
; CHECK-AVX512-O3-NEXT:    movl $3, %ecx
; CHECK-AVX512-O3-NEXT:    vzeroupper
; CHECK-AVX512-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O3-NEXT:    addq $72, %rsp
; CHECK-AVX512-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec8_double_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $72, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movapd %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT:    movapd %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movapd %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movapd %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movl $64, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $3, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT:    addq $72, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec8_double_align:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $72, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT:    vmovupd %ymm0, (%rsp)
; CHECK-AVX2-O0-NEXT:    vmovupd %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    movl $64, %edi
; CHECK-AVX2-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT:    movl $3, %ecx
; CHECK-AVX2-O0-NEXT:    vzeroupper
; CHECK-AVX2-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT:    addq $72, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec8_double_align:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $72, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    vmovupd %zmm0, (%rsp)
; CHECK-AVX512-O0-NEXT:    movl $64, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $3, %ecx
; CHECK-AVX512-O0-NEXT:    vzeroupper
; CHECK-AVX512-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT:    addq $72, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  store atomic <8 x double> %v, ptr %x release, align 64
  ret void
}

define void @store_atomic_vec16_bfloat_align(ptr %x, <16 x bfloat> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $40, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $32, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    addq $40, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    subq $40, %rsp
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $32, %edi
; CHECK-AVX-O3-NEXT:    movl $3, %ecx
; CHECK-AVX-O3-NEXT:    vzeroupper
; CHECK-AVX-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT:    addq $40, %rsp
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    subq $120, %rsp
; CHECK-SSE2-O0-NEXT:    movaps %xmm1, %xmm2
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE2-O0-NEXT:    xorps %xmm13, %xmm13
; CHECK-SSE2-O0-NEXT:    movaps %xmm13, %xmm15
; CHECK-SSE2-O0-NEXT:    punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
; CHECK-SSE2-O0-NEXT:    punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]
; CHECK-SSE2-O0-NEXT:    movaps %xmm2, %xmm14
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm14
; CHECK-SSE2-O0-NEXT:    movaps %xmm1, %xmm0
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm0
; CHECK-SSE2-O0-NEXT:    pextrw $7, %xmm2, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm12
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm12
; CHECK-SSE2-O0-NEXT:    pextrw $6, %xmm2, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm11
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm11
; CHECK-SSE2-O0-NEXT:    pextrw $5, %xmm2, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm10
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm10
; CHECK-SSE2-O0-NEXT:    pextrw $3, %xmm2, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm9
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm9
; CHECK-SSE2-O0-NEXT:    pextrw $2, %xmm2, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm8
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm8
; CHECK-SSE2-O0-NEXT:    pextrw $1, %xmm2, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm7
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm7
; CHECK-SSE2-O0-NEXT:    pextrw $7, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm6
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm6
; CHECK-SSE2-O0-NEXT:    pextrw $6, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm5
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm5
; CHECK-SSE2-O0-NEXT:    pextrw $5, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm4
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm4
; CHECK-SSE2-O0-NEXT:    pextrw $3, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm3
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm3
; CHECK-SSE2-O0-NEXT:    pextrw $2, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm2
; CHECK-SSE2-O0-NEXT:    pextrw $1, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT:    pslld $16, %xmm1
; CHECK-SSE2-O0-NEXT:    movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    movl $32, %edi
; CHECK-SSE2-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
; CHECK-SSE2-O0-NEXT:    movl $3, %ecx
; CHECK-SSE2-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE2-O0-NEXT:    addq $120, %rsp
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    subq $120, %rsp
; CHECK-SSE4-O0-NEXT:    movaps %xmm1, %xmm2
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE4-O0-NEXT:    xorps %xmm0, %xmm0
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm15
; CHECK-SSE4-O0-NEXT:    punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7]
; CHECK-SSE4-O0-NEXT:    movaps %xmm2, %xmm14
; CHECK-SSE4-O0-NEXT:    pblendw {{.*#+}} xmm14 = xmm0[0],xmm14[1],xmm0[2],xmm14[3],xmm0[4],xmm14[5],xmm0[6],xmm14[7]
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm12
; CHECK-SSE4-O0-NEXT:    punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm1[4],xmm12[5],xmm1[5],xmm12[6],xmm1[6],xmm12[7],xmm1[7]
; CHECK-SSE4-O0-NEXT:    movaps %xmm1, %xmm11
; CHECK-SSE4-O0-NEXT:    pblendw {{.*#+}} xmm11 = xmm0[0],xmm11[1],xmm0[2],xmm11[3],xmm0[4],xmm11[5],xmm0[6],xmm11[7]
; CHECK-SSE4-O0-NEXT:    movaps %xmm2, %xmm13
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm13
; CHECK-SSE4-O0-NEXT:    movaps %xmm1, %xmm0
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm0
; CHECK-SSE4-O0-NEXT:    pextrw $7, %xmm2, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm10
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm10
; CHECK-SSE4-O0-NEXT:    pextrw $6, %xmm2, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm9
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm9
; CHECK-SSE4-O0-NEXT:    pextrw $5, %xmm2, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm8
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm8
; CHECK-SSE4-O0-NEXT:    pextrw $3, %xmm2, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm7
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm7
; CHECK-SSE4-O0-NEXT:    pextrw $2, %xmm2, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm6
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm6
; CHECK-SSE4-O0-NEXT:    pextrw $7, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm5
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm5
; CHECK-SSE4-O0-NEXT:    pextrw $6, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm4
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm4
; CHECK-SSE4-O0-NEXT:    pextrw $5, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm3
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm3
; CHECK-SSE4-O0-NEXT:    pextrw $3, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm2
; CHECK-SSE4-O0-NEXT:    pextrw $2, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT:    pslld $16, %xmm1
; CHECK-SSE4-O0-NEXT:    movss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    movl $32, %edi
; CHECK-SSE4-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
; CHECK-SSE4-O0-NEXT:    movl $3, %ecx
; CHECK-SSE4-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE4-O0-NEXT:    addq $120, %rsp
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $120, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-AVX2-O0-NEXT:    vextracti128 $1, %ymm0, %xmm8
; CHECK-AVX2-O0-NEXT:    vmovdqa {{.*#+}} xmm6 = [128,128,14,15,u,u,u,u,u,u,u,u,u,u,u,u]
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm6, %xmm8, %xmm15
; CHECK-AVX2-O0-NEXT:    vmovdqa {{.*#+}} xmm5 = [128,128,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm5, %xmm8, %xmm14
; CHECK-AVX2-O0-NEXT:    vmovdqa {{.*#+}} xmm4 = [128,128,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm4, %xmm8, %xmm13
; CHECK-AVX2-O0-NEXT:    vxorps %xmm1, %xmm1, %xmm1
; CHECK-AVX2-O0-NEXT:    vpunpckhwd {{.*#+}} xmm12 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]
; CHECK-AVX2-O0-NEXT:    vmovdqa {{.*#+}} xmm3 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm3, %xmm8, %xmm11
; CHECK-AVX2-O0-NEXT:    vmovdqa {{.*#+}} xmm2 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm2, %xmm8, %xmm10
; CHECK-AVX2-O0-NEXT:    vpblendw {{.*#+}} xmm9 = xmm1[0],xmm8[1],xmm1[2],xmm8[3],xmm1[4],xmm8[5],xmm1[6],xmm8[7]
; CHECK-AVX2-O0-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm6, %xmm0, %xmm7
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm5, %xmm0, %xmm6
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm4, %xmm0, %xmm5
; CHECK-AVX2-O0-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm3, %xmm0, %xmm3
; CHECK-AVX2-O0-NEXT:    vpshufb %xmm2, %xmm0, %xmm2
; CHECK-AVX2-O0-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
; CHECK-AVX2-O0-NEXT:    vpslld $16, %xmm8, %xmm8
; CHECK-AVX2-O0-NEXT:    vpslld $16, %xmm0, %xmm0
; CHECK-AVX2-O0-NEXT:    vmovss %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vzeroupper
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    movl $32, %edi
; CHECK-AVX2-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
; CHECK-AVX2-O0-NEXT:    movl $3, %ecx
; CHECK-AVX2-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT:    addq $120, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec16_bfloat_align:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $40, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX512-O0-NEXT:    movl $32, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $3, %ecx
; CHECK-AVX512-O0-NEXT:    vzeroupper
; CHECK-AVX512-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT:    addq $40, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  store atomic <16 x bfloat> %v, ptr %x release, align 32
  ret void
}

define void @store_atomic_vec32_half_align(ptr %x, <32 x half> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec32_half_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $72, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $64, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    addq $72, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX2-O3-LABEL: store_atomic_vec32_half_align:
; CHECK-AVX2-O3:       # %bb.0:
; CHECK-AVX2-O3-NEXT:    subq $72, %rsp
; CHECK-AVX2-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O3-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX2-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT:    movl $64, %edi
; CHECK-AVX2-O3-NEXT:    movl $3, %ecx
; CHECK-AVX2-O3-NEXT:    vzeroupper
; CHECK-AVX2-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O3-NEXT:    addq $72, %rsp
; CHECK-AVX2-O3-NEXT:    retq
;
; CHECK-AVX512-O3-LABEL: store_atomic_vec32_half_align:
; CHECK-AVX512-O3:       # %bb.0:
; CHECK-AVX512-O3-NEXT:    subq $72, %rsp
; CHECK-AVX512-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT:    vmovups %zmm0, (%rsp)
; CHECK-AVX512-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT:    movl $64, %edi
; CHECK-AVX512-O3-NEXT:    movl $3, %ecx
; CHECK-AVX512-O3-NEXT:    vzeroupper
; CHECK-AVX512-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O3-NEXT:    addq $72, %rsp
; CHECK-AVX512-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec32_half_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $72, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movl $64, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $3, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT:    addq $72, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec32_half_align:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $72, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX2-O0-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    movl $64, %edi
; CHECK-AVX2-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT:    movl $3, %ecx
; CHECK-AVX2-O0-NEXT:    vzeroupper
; CHECK-AVX2-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT:    addq $72, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec32_half_align:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $72, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    vmovups %zmm0, (%rsp)
; CHECK-AVX512-O0-NEXT:    movl $64, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $3, %ecx
; CHECK-AVX512-O0-NEXT:    vzeroupper
; CHECK-AVX512-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT:    addq $72, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  store atomic <32 x half> %v, ptr %x release, align 64
  ret void
}