llvm/test/CodeGen/AMDGPU/load-atomic-local.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu6.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-SDAG %s
; RUN: llc -mtriple=amdgpu6.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-GISEL %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s

define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_f32x2_monotonic_agent:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b64 v[2:3], v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX6-NEXT:    ds_write_b32 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b64 v[2:3], v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX9-NEXT:    ds_write_b32 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f32x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b64 v[2:3], v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX10-NEXT:    ds_write_b32 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f32x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b64 v[2:3], v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX11-NEXT:    ds_store_b32 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f32x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b64 v[2:3], v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX12-NEXT:    ds_store_b32 v1, v0
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x float>, ptr addrspace(3) %p syncscope("agent") monotonic, align 8
  %num1 = extractelement <2 x float> %a0, i32 0
  %num2 = extractelement <2 x float> %a0, i32 1
  %res = fadd float %num1, %num2
  store float %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f32x2_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b64 v[2:3], v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX6-NEXT:    ds_write_b32 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b64 v[2:3], v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX9-NEXT:    ds_write_b32 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b64 v[2:3], v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX10-NEXT:    ds_write_b32 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b64 v[2:3], v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX11-NEXT:    ds_store_b32 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b64 v[2:3], v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX12-NEXT:    ds_store_b32 v1, v0
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x float>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 8
  %num1 = extractelement <2 x float> %a0, i32 0
  %num2 = extractelement <2 x float> %a0, i32 1
  %res = fadd float %num1, %num2
  store float %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f32x2_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_f32x2_monotonic_wavefront:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b64 v[2:3], v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX6-NEXT:    ds_write_b32 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f32x2_monotonic_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b64 v[2:3], v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX9-NEXT:    ds_write_b32 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f32x2_monotonic_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b64 v[2:3], v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX10-NEXT:    ds_write_b32 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f32x2_monotonic_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b64 v[2:3], v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX11-NEXT:    ds_store_b32 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f32x2_monotonic_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b64 v[2:3], v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX12-NEXT:    ds_store_b32 v1, v0
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x float>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 8
  %num1 = extractelement <2 x float> %a0, i32 0
  %num2 = extractelement <2 x float> %a0, i32 1
  %res = fadd float %num1, %num2
  store float %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 m0, -1
; GFX6-SDAG-NEXT:    ds_read_b32 v0, v0
; GFX6-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v2, v2
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-SDAG-NEXT:    ds_write_b16 v1, v0
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 m0, -1
; GFX6-GISEL-NEXT:    ds_read_b32 v0, v0
; GFX6-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v2, v2
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v2
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-GISEL-NEXT:    ds_write_b16 v1, v0
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b32 v0, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    ds_write_b16 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b32 v0, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    ds_write_b16 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b32 v0, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    ds_store_b16 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b32 v0, v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x half>, ptr addrspace(3) %p syncscope("agent") monotonic, align 4
  %num1 = extractelement <2 x half> %a0, i32 0
  %num2 = extractelement <2 x half> %a0, i32 1
  %res = fadd half %num1, %num2
  store half %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f16x2_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 m0, -1
; GFX6-SDAG-NEXT:    ds_read_b32 v0, v0
; GFX6-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v2, v2
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-SDAG-NEXT:    ds_write_b16 v1, v0
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 m0, -1
; GFX6-GISEL-NEXT:    ds_read_b32 v0, v0
; GFX6-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v2, v2
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v2
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-GISEL-NEXT:    ds_write_b16 v1, v0
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f16x2_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b32 v0, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    ds_write_b16 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f16x2_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b32 v0, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    ds_write_b16 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f16x2_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b32 v0, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    ds_store_b16 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f16x2_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b32 v0, v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x half>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 4
  %num1 = extractelement <2 x half> %a0, i32 0
  %num2 = extractelement <2 x half> %a0, i32 1
  %res = fadd half %num1, %num2
  store half %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f16x2_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 m0, -1
; GFX6-SDAG-NEXT:    ds_read_b32 v0, v0
; GFX6-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v2, v2
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v2
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-SDAG-NEXT:    ds_write_b16 v1, v0
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 m0, -1
; GFX6-GISEL-NEXT:    ds_read_b32 v0, v0
; GFX6-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v2, v2
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v2
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-GISEL-NEXT:    ds_write_b16 v1, v0
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f16x2_monotonic_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b32 v0, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    ds_write_b16 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f16x2_monotonic_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b32 v0, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    ds_write_b16 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f16x2_monotonic_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b32 v0, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    ds_store_b16 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f16x2_monotonic_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b32 v0, v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x half>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 4
  %num1 = extractelement <2 x half> %a0, i32 0
  %num2 = extractelement <2 x half> %a0, i32 1
  %res = fadd half %num1, %num2
  store half %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_i16x2_monotonic_agent:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b32 v0, v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
; GFX6-NEXT:    ds_write_b16 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b32 v0, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    ds_write_b16 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b32 v0, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_add_nc_u16 v0, v0, v0 op_sel:[0,1,0]
; GFX10-NEXT:    ds_write_b16 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b32 v0, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX11-NEXT:    ds_store_b16 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b32 v0, v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  %a = load atomic <2 x i16>, ptr addrspace(3) %p syncscope("agent") monotonic, align 4
  %e0 = extractelement <2 x i16> %a, i32 0
  %e1 = extractelement <2 x i16> %a, i32 1
  %sum = add i16 %e0, %e1
  store i16 %sum, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x2_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_i16x2_seq_cst_agent:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b32 v0, v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
; GFX6-NEXT:    ds_write_b16 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x2_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b32 v0, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    ds_write_b16 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x2_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b32 v0, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    v_add_nc_u16 v0, v0, v0 op_sel:[0,1,0]
; GFX10-NEXT:    ds_write_b16 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x2_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b32 v0, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX11-NEXT:    ds_store_b16 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x2_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b32 v0, v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  %a = load atomic <2 x i16>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 4
  %e0 = extractelement <2 x i16> %a, i32 0
  %e1 = extractelement <2 x i16> %a, i32 1
  %sum = add i16 %e0, %e1
  store i16 %sum, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x2_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_i16x2_monotonic_wavefront:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b32 v0, v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
; GFX6-NEXT:    ds_write_b16 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x2_monotonic_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b32 v0, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    ds_write_b16 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x2_monotonic_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b32 v0, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_add_nc_u16 v0, v0, v0 op_sel:[0,1,0]
; GFX10-NEXT:    ds_write_b16 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x2_monotonic_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b32 v0, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX11-NEXT:    ds_store_b16 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x2_monotonic_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b32 v0, v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  %a = load atomic <2 x i16>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 4
  %e0 = extractelement <2 x i16> %a, i32 0
  %e1 = extractelement <2 x i16> %a, i32 1
  %sum = add i16 %e0, %e1
  store i16 %sum, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_f16x4_monotonic_agent:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b64 v[2:3], v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT:    v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT:    v_cvt_f32_f16_e32 v4, v4
; GFX6-NEXT:    v_add_f32_e32 v0, v2, v0
; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-NEXT:    v_mul_f32_e32 v2, v3, v4
; GFX6-NEXT:    v_cvt_f16_f32_e32 v2, v2
; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT:    v_add_f32_e32 v0, v0, v2
; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-NEXT:    ds_write_b16 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f16x4_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b64 v[2:3], v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_add_f16_e32 v0, v0, v2
; GFX9-NEXT:    ds_write_b16 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f16x4_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b64 v[2:3], v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    v_add_f16_e32 v0, v0, v2
; GFX10-NEXT:    ds_write_b16 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f16x4_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b64 v[2:3], v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_f16_e32 v0.l, v2.l, v2.h
; GFX11-NEXT:    v_mul_f16_e32 v0.h, v3.l, v3.h
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    ds_store_b16 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f16x4_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b64 v[2:3], v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_f16_e32 v0.l, v2.l, v2.h
; GFX12-NEXT:    v_mul_f16_e32 v0.h, v3.l, v3.h
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <4 x half>, ptr addrspace(3) %p syncscope("agent") monotonic, align 8
  %num1 = extractelement <4 x half> %a0, i32 0
  %num2 = extractelement <4 x half> %a0, i32 1
  %num3 = extractelement <4 x half> %a0, i32 2
  %num4 = extractelement <4 x half> %a0, i32 3
  %add = fadd half %num1, %num2
  %mul = fmul half %num3, %num4
  %res = fadd half %add, %mul
  store half %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f16x4_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_f16x4_seq_cst_agent:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b64 v[2:3], v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT:    v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT:    v_cvt_f32_f16_e32 v4, v4
; GFX6-NEXT:    v_add_f32_e32 v0, v2, v0
; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-NEXT:    v_mul_f32_e32 v2, v3, v4
; GFX6-NEXT:    v_cvt_f16_f32_e32 v2, v2
; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT:    v_add_f32_e32 v0, v0, v2
; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-NEXT:    ds_write_b16 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f16x4_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b64 v[2:3], v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_add_f16_e32 v0, v0, v2
; GFX9-NEXT:    ds_write_b16 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f16x4_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b64 v[2:3], v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    v_add_f16_e32 v0, v0, v2
; GFX10-NEXT:    ds_write_b16 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f16x4_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b64 v[2:3], v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    v_add_f16_e32 v0.l, v2.l, v2.h
; GFX11-NEXT:    v_mul_f16_e32 v0.h, v3.l, v3.h
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    ds_store_b16 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f16x4_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b64 v[2:3], v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    v_add_f16_e32 v0.l, v2.l, v2.h
; GFX12-NEXT:    v_mul_f16_e32 v0.h, v3.l, v3.h
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <4 x half>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 8
  %num1 = extractelement <4 x half> %a0, i32 0
  %num2 = extractelement <4 x half> %a0, i32 1
  %num3 = extractelement <4 x half> %a0, i32 2
  %num4 = extractelement <4 x half> %a0, i32 3
  %add = fadd half %num1, %num2
  %mul = fmul half %num3, %num4
  %res = fadd half %add, %mul
  store half %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f16x4_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_f16x4_monotonic_wavefront:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b64 v[2:3], v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT:    v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT:    v_cvt_f32_f16_e32 v4, v4
; GFX6-NEXT:    v_add_f32_e32 v0, v2, v0
; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-NEXT:    v_mul_f32_e32 v2, v3, v4
; GFX6-NEXT:    v_cvt_f16_f32_e32 v2, v2
; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT:    v_add_f32_e32 v0, v0, v2
; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-NEXT:    ds_write_b16 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f16x4_monotonic_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b64 v[2:3], v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_add_f16_e32 v0, v0, v2
; GFX9-NEXT:    ds_write_b16 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f16x4_monotonic_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b64 v[2:3], v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_add_f16_sdwa v0, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    v_mul_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    v_add_f16_e32 v0, v0, v2
; GFX10-NEXT:    ds_write_b16 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f16x4_monotonic_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b64 v[2:3], v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_f16_e32 v0.l, v2.l, v2.h
; GFX11-NEXT:    v_mul_f16_e32 v0.h, v3.l, v3.h
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    ds_store_b16 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f16x4_monotonic_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b64 v[2:3], v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_f16_e32 v0.l, v2.l, v2.h
; GFX12-NEXT:    v_mul_f16_e32 v0.h, v3.l, v3.h
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <4 x half>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 8
  %num1 = extractelement <4 x half> %a0, i32 0
  %num2 = extractelement <4 x half> %a0, i32 1
  %num3 = extractelement <4 x half> %a0, i32 2
  %num4 = extractelement <4 x half> %a0, i32 3
  %add = fadd half %num1, %num2
  %mul = fmul half %num3, %num4
  %res = fadd half %add, %mul
  store half %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_i16x4_monotonic_agent:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b64 v[2:3], v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
; GFX6-NEXT:    v_and_b32_e32 v2, 0xffff, v3
; GFX6-NEXT:    v_mad_u32_u24 v0, v2, v4, v0
; GFX6-NEXT:    ds_write_b16 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x4_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b64 v[2:3], v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v3
; GFX9-NEXT:    v_add_u16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_mad_legacy_u16 v0, v3, v0, v2
; GFX9-NEXT:    ds_write_b16 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x4_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b64 v[2:3], v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v3
; GFX10-NEXT:    v_add_nc_u16 v2, v2, v2 op_sel:[0,1,0]
; GFX10-NEXT:    v_mad_u16 v0, v3, v0, v2
; GFX10-NEXT:    ds_write_b16 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x4_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b64 v[2:3], v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_nc_u16 v0.l, v2.l, v2.h
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_mad_u16 v0.l, v3.l, v3.h, v0.l
; GFX11-NEXT:    ds_store_b16 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x4_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b64 v[2:3], v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_nc_u16 v0.l, v2.l, v2.h
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_mad_u16 v0.l, v3.l, v3.h, v0.l
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <4 x i16>, ptr addrspace(3) %p syncscope("agent") monotonic, align 8
  %num1 = extractelement <4 x i16> %a0, i32 0
  %num2 = extractelement <4 x i16> %a0, i32 1
  %num3 = extractelement <4 x i16> %a0, i32 2
  %num4 = extractelement <4 x i16> %a0, i32 3
  %add = add i16 %num1, %num2
  %mul = mul i16 %num3, %num4
  %res = add i16 %add, %mul
  store i16 %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x4_seq_cst_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_i16x4_seq_cst_agent:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b64 v[2:3], v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
; GFX6-NEXT:    v_and_b32_e32 v2, 0xffff, v3
; GFX6-NEXT:    v_mad_u32_u24 v0, v2, v4, v0
; GFX6-NEXT:    ds_write_b16 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x4_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b64 v[2:3], v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v3
; GFX9-NEXT:    v_add_u16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_mad_legacy_u16 v0, v3, v0, v2
; GFX9-NEXT:    ds_write_b16 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x4_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b64 v[2:3], v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v3
; GFX10-NEXT:    v_add_nc_u16 v2, v2, v2 op_sel:[0,1,0]
; GFX10-NEXT:    v_mad_u16 v0, v3, v0, v2
; GFX10-NEXT:    ds_write_b16 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x4_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b64 v[2:3], v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    v_add_nc_u16 v0.l, v2.l, v2.h
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_mad_u16 v0.l, v3.l, v3.h, v0.l
; GFX11-NEXT:    ds_store_b16 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x4_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b64 v[2:3], v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    v_add_nc_u16 v0.l, v2.l, v2.h
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_mad_u16 v0.l, v3.l, v3.h, v0.l
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <4 x i16>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 8
  %num1 = extractelement <4 x i16> %a0, i32 0
  %num2 = extractelement <4 x i16> %a0, i32 1
  %num3 = extractelement <4 x i16> %a0, i32 2
  %num4 = extractelement <4 x i16> %a0, i32 3
  %add = add i16 %num1, %num2
  %mul = mul i16 %num3, %num4
  %res = add i16 %add, %mul
  store i16 %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x4_monotonic_wavefront(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_i16x4_monotonic_wavefront:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b64 v[2:3], v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
; GFX6-NEXT:    v_and_b32_e32 v2, 0xffff, v3
; GFX6-NEXT:    v_mad_u32_u24 v0, v2, v4, v0
; GFX6-NEXT:    ds_write_b16 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x4_monotonic_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b64 v[2:3], v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v3
; GFX9-NEXT:    v_add_u16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_mad_legacy_u16 v0, v3, v0, v2
; GFX9-NEXT:    ds_write_b16 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x4_monotonic_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b64 v[2:3], v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v3
; GFX10-NEXT:    v_add_nc_u16 v2, v2, v2 op_sel:[0,1,0]
; GFX10-NEXT:    v_mad_u16 v0, v3, v0, v2
; GFX10-NEXT:    ds_write_b16 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x4_monotonic_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b64 v[2:3], v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_nc_u16 v0.l, v2.l, v2.h
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_mad_u16 v0.l, v3.l, v3.h, v0.l
; GFX11-NEXT:    ds_store_b16 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x4_monotonic_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b64 v[2:3], v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_nc_u16 v0.l, v2.l, v2.h
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_mad_u16 v0.l, v3.l, v3.h, v0.l
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <4 x i16>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 8
  %num1 = extractelement <4 x i16> %a0, i32 0
  %num2 = extractelement <4 x i16> %a0, i32 1
  %num3 = extractelement <4 x i16> %a0, i32 2
  %num4 = extractelement <4 x i16> %a0, i32 3
  %add = add i16 %num1, %num2
  %mul = mul i16 %num3, %num4
  %res = add i16 %add, %mul
  store i16 %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_1(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 1, v0
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b64 v[2:3], v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX6-NEXT:    ds_write_b32 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b64 v[2:3], v0 offset:1
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX9-NEXT:    ds_write_b32 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b64 v[2:3], v0 offset:1
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX10-NEXT:    ds_write_b32 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b64 v[2:3], v0 offset:1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX11-NEXT:    ds_store_b32 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b64 v[2:3], v0 offset:1
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX12-NEXT:    ds_store_b32 v1, v0
; GFX12-NEXT:    s_endpgm
  %gep = getelementptr inbounds i8, ptr addrspace(3) %p, i64 1
  %a0 = load atomic <2 x float>, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
  %num1 = extractelement <2 x float> %a0, i32 0
  %num2 = extractelement <2 x float> %a0, i32 1
  %res = fadd float %num1, %num2
  store float %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_max(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 0xfff, v0
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b64 v[2:3], v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX6-NEXT:    ds_write_b32 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b64 v[2:3], v0 offset:4095
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX9-NEXT:    ds_write_b32 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b64 v[2:3], v0 offset:4095
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX10-NEXT:    ds_write_b32 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b64 v[2:3], v0 offset:4095
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX11-NEXT:    ds_store_b32 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b64 v[2:3], v0 offset:4095
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_f32_e32 v0, v2, v3
; GFX12-NEXT:    ds_store_b32 v1, v0
; GFX12-NEXT:    s_endpgm
  %gep = getelementptr inbounds i8, ptr addrspace(3) %p, i64 4095
  %a0 = load atomic <2 x float>, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
  %num1 = extractelement <2 x float> %a0, i32 0
  %num2 = extractelement <2 x float> %a0, i32 1
  %res = fadd float %num1, %num2
  store float %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_1(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 1, v0
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b32 v0, v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
; GFX6-NEXT:    ds_write_b16 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b32 v0, v0 offset:1
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    ds_write_b16 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b32 v0, v0 offset:1
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_add_nc_u16 v0, v0, v0 op_sel:[0,1,0]
; GFX10-NEXT:    ds_write_b16 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b32 v0, v0 offset:1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX11-NEXT:    ds_store_b16 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b32 v0, v0 offset:1
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  %gep = getelementptr inbounds i8, ptr addrspace(3) %p, i64 1
  %a = load atomic <2 x i16>, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
  %e0 = extractelement <2 x i16> %a, i32 0
  %e1 = extractelement <2 x i16> %a, i32 1
  %sum = add i16 %e0, %e1
  store i16 %sum, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 0xfff, v0
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b32 v0, v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
; GFX6-NEXT:    ds_write_b16 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b32 v0, v0 offset:4095
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    ds_write_b16 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b32 v0, v0 offset:4095
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_add_nc_u16 v0, v0, v0 op_sel:[0,1,0]
; GFX10-NEXT:    ds_write_b16 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b32 v0, v0 offset:4095
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX11-NEXT:    ds_store_b16 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b32 v0, v0 offset:4095
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX12-NEXT:    ds_store_b16 v1, v0
; GFX12-NEXT:    s_endpgm
  %gep = getelementptr inbounds i8, ptr addrspace(3) %p, i64 4095
  %a = load atomic <2 x i16>, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
  %e0 = extractelement <2 x i16> %a, i32 0
  %e1 = extractelement <2 x i16> %a, i32 1
  %sum = add i16 %e0, %e1
  store i16 %sum, ptr addrspace(3) %out, align 4
  ret void
}
define amdgpu_cs void @atomic_load_i32x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_i32x2_monotonic_agent:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b64 v[2:3], v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v2, v3
; GFX6-NEXT:    ds_write_b32 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i32x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b64 v[2:3], v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_u32_e32 v0, v2, v3
; GFX9-NEXT:    ds_write_b32 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i32x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b64 v[2:3], v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_add_nc_u32_e32 v0, v2, v3
; GFX10-NEXT:    ds_write_b32 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i32x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b64 v[2:3], v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_add_nc_u32_e32 v0, v2, v3
; GFX11-NEXT:    ds_store_b32 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i32x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b64 v[2:3], v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    v_add_nc_u32_e32 v0, v2, v3
; GFX12-NEXT:    ds_store_b32 v1, v0
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x i32>, ptr addrspace(3) %p syncscope("agent") monotonic, align 8
  %num1 = extractelement <2 x i32> %a0, i32 0
  %num2 = extractelement <2 x i32> %a0, i32 1
  %res = add i32 %num1, %num2
  store i32 %res, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_bf16x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_bf16x2_monotonic_agent:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b32 v0, v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    ds_write_b32 v1, v0
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_bf16x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b32 v0, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    ds_write_b32 v1, v0
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_bf16x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b32 v0, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    ds_write_b32 v1, v0
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_bf16x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b32 v0, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    ds_store_b32 v1, v0
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_bf16x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b32 v0, v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    ds_store_b32 v1, v0
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x bfloat>, ptr addrspace(3) %p syncscope("agent") monotonic, align 4
  store <2 x bfloat> %a0, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_bf16x4_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX6-LABEL: atomic_load_bf16x4_monotonic_agent:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 m0, -1
; GFX6-NEXT:    ds_read_b64 v[2:3], v0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    ds_write_b64 v1, v[2:3]
; GFX6-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_bf16x4_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    ds_read_b64 v[2:3], v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    ds_write_b64 v1, v[2:3]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_bf16x4_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    ds_read_b64 v[2:3], v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    ds_write_b64 v1, v[2:3]
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_bf16x4_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    ds_load_b64 v[2:3], v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    ds_store_b64 v1, v[2:3]
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_bf16x4_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    ds_load_b64 v[2:3], v0
; GFX12-NEXT:    s_wait_dscnt 0x0
; GFX12-NEXT:    ds_store_b64 v1, v[2:3]
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <4 x bfloat>, ptr addrspace(3) %p syncscope("agent") monotonic, align 8
  store <4 x bfloat> %a0, ptr addrspace(3) %out, align 8
  ret void
}

;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
; GFX10-GISEL: {{.*}}
; GFX10-SDAG: {{.*}}
; GFX11-GISEL: {{.*}}
; GFX11-SDAG: {{.*}}
; GFX12-GISEL: {{.*}}
; GFX12-SDAG: {{.*}}
; GFX9-GISEL: {{.*}}
; GFX9-SDAG: {{.*}}