llvm/test/CodeGen/AMDGPU/store-atomic-global.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu6.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-SDAG %s
; RUN: llc -mtriple=amdgpu6.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-GISEL %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s

define amdgpu_cs void @atomic_store_f32x2_monotonic_agent(<2 x float> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_f32x2_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_f32x2_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_f32x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_f32x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_f32x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f32x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
  ret void
}

define amdgpu_cs void @atomic_store_f32x2_seq_cst_agent(<2 x float> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_f32x2_seq_cst_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_f32x2_seq_cst_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_f32x2_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_f32x2_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_f32x2_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f32x2_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
  ret void
}

define amdgpu_cs void @atomic_store_f32x2_seq_cst_wavefront(<2 x float> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_f32x2_seq_cst_wavefront:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_f32x2_seq_cst_wavefront:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_f32x2_seq_cst_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_f32x2_seq_cst_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_f32x2_seq_cst_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f32x2_seq_cst_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX12-NEXT:    s_endpgm
  store atomic <2 x float> %in, ptr addrspace(1) %out syncscope("wavefront") seq_cst, align 8
  ret void
}

define amdgpu_cs void @atomic_store_f16x2_monotonic_agent(<2 x half> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_f16x2_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_f16x2_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_f16x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dword v[1:2], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_f16x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dword v[1:2], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_f16x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b32 v[1:2], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f16x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
  ret void
}

define amdgpu_cs void @atomic_store_f16x2_seq_cst_agent(<2 x half> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_f16x2_seq_cst_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_f16x2_seq_cst_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_f16x2_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dword v[1:2], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_f16x2_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dword v[1:2], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_f16x2_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b32 v[1:2], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f16x2_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
  ret void
}

define amdgpu_cs void @atomic_store_f16x2_monotonic_wavefront(<2 x half> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_f16x2_monotonic_wavefront:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_f16x2_monotonic_wavefront:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_f16x2_monotonic_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dword v[1:2], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_f16x2_monotonic_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dword v[1:2], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_f16x2_monotonic_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b32 v[1:2], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f16x2_monotonic_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b32 v[1:2], v0, off
; GFX12-NEXT:    s_endpgm
  store atomic <2 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
  ret void
}

define amdgpu_cs void @atomic_store_i16x2_monotonic_agent(<2 x i16> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_i16x2_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_i16x2_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_i16x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dword v[1:2], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_i16x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dword v[1:2], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_i16x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b32 v[1:2], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i16x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
  ret void
}

define amdgpu_cs void @atomic_store_i16x2_seq_cst_agent(<2 x i16> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_i16x2_seq_cst_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_i16x2_seq_cst_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_i16x2_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dword v[1:2], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_i16x2_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dword v[1:2], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_i16x2_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b32 v[1:2], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i16x2_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
  ret void
}

define amdgpu_cs void @atomic_store_i16x2_monotonic_wavefront(<2 x i16> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_i16x2_monotonic_wavefront:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_i16x2_monotonic_wavefront:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_i16x2_monotonic_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dword v[1:2], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_i16x2_monotonic_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dword v[1:2], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_i16x2_monotonic_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b32 v[1:2], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i16x2_monotonic_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b32 v[1:2], v0, off
; GFX12-NEXT:    s_endpgm
  store atomic <2 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
  ret void
}

define amdgpu_cs void @atomic_store_f16x4_monotonic_agent(<4 x half> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_f16x4_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_f16x4_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
; GFX6-GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
; GFX6-GISEL-NEXT:    v_or_b32_e32 v0, v4, v0
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
; GFX6-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-GISEL-NEXT:    v_or_b32_e32 v1, v4, v1
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_f16x4_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_f16x4_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_f16x4_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f16x4_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
  ret void
}

define amdgpu_cs void @atomic_store_f16x4_seq_cst_agent(<4 x half> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_f16x4_seq_cst_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_f16x4_seq_cst_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
; GFX6-GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
; GFX6-GISEL-NEXT:    v_or_b32_e32 v0, v4, v0
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
; GFX6-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-GISEL-NEXT:    v_or_b32_e32 v1, v4, v1
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_f16x4_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_f16x4_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_f16x4_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f16x4_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
  ret void
}

define amdgpu_cs void @atomic_store_f16x4_monotonic_wavefront(<4 x half> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_f16x4_monotonic_wavefront:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_f16x4_monotonic_wavefront:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
; GFX6-GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
; GFX6-GISEL-NEXT:    v_or_b32_e32 v0, v4, v0
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
; GFX6-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-GISEL-NEXT:    v_or_b32_e32 v1, v4, v1
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_f16x4_monotonic_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_f16x4_monotonic_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_f16x4_monotonic_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f16x4_monotonic_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX12-NEXT:    s_endpgm
  store atomic <4 x half> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
  ret void
}

define amdgpu_cs void @atomic_store_i16x4_monotonic_agent(<4 x i16> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_i16x4_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_i16x4_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
; GFX6-GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
; GFX6-GISEL-NEXT:    v_or_b32_e32 v0, v4, v0
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
; GFX6-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-GISEL-NEXT:    v_or_b32_e32 v1, v4, v1
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_i16x4_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_i16x4_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_i16x4_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i16x4_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
  ret void
}

define amdgpu_cs void @atomic_store_i16x4_seq_cst_agent(<4 x i16> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_i16x4_seq_cst_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_i16x4_seq_cst_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
; GFX6-GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
; GFX6-GISEL-NEXT:    v_or_b32_e32 v0, v4, v0
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
; GFX6-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-GISEL-NEXT:    v_or_b32_e32 v1, v4, v1
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_i16x4_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_i16x4_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_i16x4_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i16x4_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 8
  ret void
}

define amdgpu_cs void @atomic_store_i16x4_monotonic_wavefront(<4 x i16> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_i16x4_monotonic_wavefront:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_i16x4_monotonic_wavefront:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
; GFX6-GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
; GFX6-GISEL-NEXT:    v_or_b32_e32 v0, v4, v0
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
; GFX6-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-GISEL-NEXT:    v_or_b32_e32 v1, v4, v1
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_i16x4_monotonic_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_i16x4_monotonic_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_i16x4_monotonic_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i16x4_monotonic_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX12-NEXT:    s_endpgm
  store atomic <4 x i16> %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 8
  ret void
}

define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_min(<4 x i16> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_movk_i32 s0, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s1, -1
; GFX6-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
; GFX6-GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
; GFX6-GISEL-NEXT:    v_or_b32_e32 v0, v4, v0
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
; GFX6-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-GISEL-NEXT:    v_or_b32_e32 v1, v4, v1
; GFX6-GISEL-NEXT:    s_movk_i32 s0, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b32 s1, -1
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off offset:-4096
; GFX9-NEXT:    s_endpgm
;
; GFX10-SDAG-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
; GFX10-SDAG:       ; %bb.0:
; GFX10-SDAG-NEXT:    v_add_co_u32 v2, vcc_lo, 0xfffff000, v2
; GFX10-SDAG-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, -1, v3, vcc_lo
; GFX10-SDAG-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-SDAG-NEXT:    s_endpgm
;
; GFX10-GISEL-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
; GFX10-GISEL:       ; %bb.0:
; GFX10-GISEL-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 0xfffff000
; GFX10-GISEL-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, -1, v3, vcc_lo
; GFX10-GISEL-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-GISEL-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off offset:-4096
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i16x4_monotonic_agent_offset_min:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off offset:-4096 scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  %gep = getelementptr inbounds i8, ptr addrspace(1) %out, i64 -4096
  store atomic <4 x i16> %in, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
  ret void
}

define amdgpu_cs void @atomic_store_i16x4_monotonic_agent_offset_max(<4 x i16> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:4088
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
; GFX6-GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
; GFX6-GISEL-NEXT:    v_or_b32_e32 v0, v4, v0
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
; GFX6-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-GISEL-NEXT:    v_or_b32_e32 v1, v4, v1
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:4088
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off offset:4088
; GFX9-NEXT:    s_endpgm
;
; GFX10-SDAG-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
; GFX10-SDAG:       ; %bb.0:
; GFX10-SDAG-NEXT:    v_add_co_u32 v2, vcc_lo, 0x800, v2
; GFX10-SDAG-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-SDAG-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off offset:2040
; GFX10-SDAG-NEXT:    s_endpgm
;
; GFX10-GISEL-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
; GFX10-GISEL:       ; %bb.0:
; GFX10-GISEL-NEXT:    v_add_co_u32 v2, vcc_lo, v2, 0xff8
; GFX10-GISEL-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX10-GISEL-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-GISEL-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off offset:4088
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i16x4_monotonic_agent_offset_max:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off offset:4088 scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  %gep = getelementptr inbounds i8, ptr addrspace(1) %out, i64 4088
  store atomic <4 x i16> %in, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
  ret void
}
define amdgpu_cs void @atomic_store_i32x2_monotonic_agent(<2 x i32> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_i32x2_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_i32x2_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_i32x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_i32x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_i32x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i32x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  store atomic <2 x i32> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
  ret void
}

define amdgpu_cs void @atomic_store_bf16x2_monotonic_agent(<2 x bfloat> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_bf16x2_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_bf16x2_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_bf16x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dword v[1:2], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_bf16x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dword v[1:2], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_bf16x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b32 v[1:2], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_bf16x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b32 v[1:2], v0, off scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  store atomic <2 x bfloat> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
  ret void
}

define amdgpu_cs void @atomic_store_bf16x4_monotonic_agent(<4 x bfloat> %in, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_store_bf16x4_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_store_bf16x4_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
; GFX6-GISEL-NEXT:    v_and_b32_e32 v0, 0xffff, v0
; GFX6-GISEL-NEXT:    v_or_b32_e32 v0, v4, v0
; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
; GFX6-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-GISEL-NEXT:    v_or_b32_e32 v1, v4, v1
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_store_bf16x4_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_store_bf16x4_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_store_bf16x4_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_bf16x4_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_endpgm
  store atomic <4 x bfloat> %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 8
  ret void
}

;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
; GFX11-GISEL: {{.*}}
; GFX11-SDAG: {{.*}}
; GFX12-GISEL: {{.*}}
; GFX12-SDAG: {{.*}}
; GFX6: {{.*}}
; GFX9-GISEL: {{.*}}
; GFX9-SDAG: {{.*}}