llvm/test/CodeGen/AMDGPU/load-atomic-global.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu6.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-SDAG %s
; RUN: llc -mtriple=amdgpu6.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-GISEL %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s

define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_f32x2_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-SDAG-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_f32x2_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-GISEL-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX9-NEXT:    global_store_dword v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f32x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX10-NEXT:    global_store_dword v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f32x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX11-NEXT:    global_store_b32 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f32x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX12-NEXT:    global_store_b32 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
  %num1 = extractelement <2 x float> %a0, i32 0
  %num2 = extractelement <2 x float> %a0, i32 1
  %res = fadd float %num1, %num2
  store float %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f32x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    buffer_wbinvl1
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-SDAG-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    buffer_wbinvl1
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-GISEL-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX9-NEXT:    global_store_dword v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX10-NEXT:    global_store_dword v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX11-NEXT:    global_store_b32 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f32x2_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX12-NEXT:    global_store_b32 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
  %num1 = extractelement <2 x float> %a0, i32 0
  %num2 = extractelement <2 x float> %a0, i32 1
  %res = fadd float %num1, %num2
  store float %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f32x2_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_f32x2_monotonic_wavefront:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-SDAG-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_f32x2_monotonic_wavefront:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-GISEL-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f32x2_monotonic_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX9-NEXT:    global_store_dword v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f32x2_monotonic_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX10-NEXT:    global_store_dword v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f32x2_monotonic_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX11-NEXT:    global_store_b32 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f32x2_monotonic_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX12-NEXT:    global_store_b32 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
  %num1 = extractelement <2 x float> %a0, i32 0
  %num2 = extractelement <2 x float> %a0, i32 1
  %res = fadd float %num1, %num2
  store float %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-SDAG-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-GISEL-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dword v0, v[0:1], off glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    global_store_short v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dword v0, v[0:1], off glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    global_store_short v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b32 v0, v[0:1], off glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    global_store_b16 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f16x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    global_store_b16 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
  %num1 = extractelement <2 x half> %a0, i32 0
  %num2 = extractelement <2 x half> %a0, i32 1
  %res = fadd half %num1, %num2
  store half %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f16x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    buffer_wbinvl1
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-SDAG-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    buffer_wbinvl1
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-GISEL-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f16x2_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dword v0, v[0:1], off glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    global_store_short v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f16x2_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dword v0, v[0:1], off glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    global_store_short v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f16x2_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b32 v0, v[0:1], off glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    global_store_b16 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f16x2_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    global_store_b16 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 4
  %num1 = extractelement <2 x half> %a0, i32 0
  %num2 = extractelement <2 x half> %a0, i32 1
  %res = fadd half %num1, %num2
  store half %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f16x2_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-SDAG-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-GISEL-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f16x2_monotonic_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dword v0, v[0:1], off
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    global_store_short v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f16x2_monotonic_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dword v0, v[0:1], off
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    global_store_short v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f16x2_monotonic_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b32 v0, v[0:1], off
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    global_store_b16 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f16x2_monotonic_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b32 v0, v[0:1], off
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    global_store_b16 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4
  %num1 = extractelement <2 x half> %a0, i32 0
  %num2 = extractelement <2 x half> %a0, i32 1
  %res = fadd half %num1, %num2
  store half %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
; GFX6-SDAG-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
; GFX6-GISEL-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dword v0, v[0:1], off glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    global_store_short v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dword v0, v[0:1], off glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_add_nc_u16 v0, v0, v0 op_sel:[0,1,0]
; GFX10-NEXT:    global_store_short v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b32 v0, v[0:1], off glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX11-NEXT:    global_store_b16 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX12-NEXT:    global_store_b16 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
  %e0 = extractelement <2 x i16> %a, i32 0
  %e1 = extractelement <2 x i16> %a, i32 1
  %sum = add i16 %e0, %e1
  store i16 %sum, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x2_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_i16x2_seq_cst_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    buffer_wbinvl1
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
; GFX6-SDAG-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_i16x2_seq_cst_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    buffer_wbinvl1
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
; GFX6-GISEL-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x2_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dword v0, v[0:1], off glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    global_store_short v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x2_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dword v0, v[0:1], off glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    v_add_nc_u16 v0, v0, v0 op_sel:[0,1,0]
; GFX10-NEXT:    global_store_short v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x2_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b32 v0, v[0:1], off glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX11-NEXT:    global_store_b16 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x2_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX12-NEXT:    global_store_b16 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 4
  %e0 = extractelement <2 x i16> %a, i32 0
  %e1 = extractelement <2 x i16> %a, i32 1
  %sum = add i16 %e0, %e1
  store i16 %sum, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x2_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_i16x2_monotonic_wavefront:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
; GFX6-SDAG-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_i16x2_monotonic_wavefront:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
; GFX6-GISEL-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x2_monotonic_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dword v0, v[0:1], off
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    global_store_short v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x2_monotonic_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dword v0, v[0:1], off
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_add_nc_u16 v0, v0, v0 op_sel:[0,1,0]
; GFX10-NEXT:    global_store_short v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x2_monotonic_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b32 v0, v[0:1], off
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX11-NEXT:    global_store_b16 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x2_monotonic_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b32 v0, v[0:1], off
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX12-NEXT:    global_store_b16 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4
  %e0 = extractelement <2 x i16> %a, i32 0
  %e1 = extractelement <2 x i16> %a, i32 1
  %sum = add i16 %e0, %e1
  store i16 %sum, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v4, v4
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v5, v5
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v4
; GFX6-SDAG-NEXT:    v_mul_f32_e32 v1, v1, v5
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v1, v1
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-SDAG-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v4, v4
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v5, v5
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v4
; GFX6-GISEL-NEXT:    v_mul_f32_e32 v1, v1, v5
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v1, v1
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-GISEL-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f16x4_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_add_f16_e32 v0, v0, v1
; GFX9-NEXT:    global_store_short v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f16x4_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    v_add_f16_e32 v0, v0, v1
; GFX10-NEXT:    global_store_short v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f16x4_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    v_mul_f16_e32 v0.h, v1.l, v1.h
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    global_store_b16 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f16x4_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    v_mul_f16_e32 v0.h, v1.l, v1.h
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    global_store_b16 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
  %num1 = extractelement <4 x half> %a0, i32 0
  %num2 = extractelement <4 x half> %a0, i32 1
  %num3 = extractelement <4 x half> %a0, i32 2
  %num4 = extractelement <4 x half> %a0, i32 3
  %add = fadd half %num1, %num2
  %mul = fmul half %num3, %num4
  %res = fadd half %add, %mul
  store half %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f16x4_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_f16x4_seq_cst_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    buffer_wbinvl1
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v4, v4
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v5, v5
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v4
; GFX6-SDAG-NEXT:    v_mul_f32_e32 v1, v1, v5
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v1, v1
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-SDAG-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_f16x4_seq_cst_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    buffer_wbinvl1
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v4, v4
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v5, v5
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v4
; GFX6-GISEL-NEXT:    v_mul_f32_e32 v1, v1, v5
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v1, v1
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-GISEL-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f16x4_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_add_f16_e32 v0, v0, v1
; GFX9-NEXT:    global_store_short v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f16x4_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    v_add_f16_e32 v0, v0, v1
; GFX10-NEXT:    global_store_short v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f16x4_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    v_mul_f16_e32 v0.h, v1.l, v1.h
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    global_store_b16 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f16x4_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    v_mul_f16_e32 v0.h, v1.l, v1.h
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    global_store_b16 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
  %num1 = extractelement <4 x half> %a0, i32 0
  %num2 = extractelement <4 x half> %a0, i32 1
  %num3 = extractelement <4 x half> %a0, i32 2
  %num4 = extractelement <4 x half> %a0, i32 3
  %add = fadd half %num1, %num2
  %mul = fmul half %num3, %num4
  %res = fadd half %add, %mul
  store half %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f16x4_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_f16x4_monotonic_wavefront:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v4, v4
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v5, v5
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v4
; GFX6-SDAG-NEXT:    v_mul_f32_e32 v1, v1, v5
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v1, v1
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-SDAG-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-SDAG-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-SDAG-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_f16x4_monotonic_wavefront:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v4, v4
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v5, v5
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v4
; GFX6-GISEL-NEXT:    v_mul_f32_e32 v1, v1, v5
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v1, v1
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v0, v0
; GFX6-GISEL-NEXT:    v_cvt_f32_f16_e32 v1, v1
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-GISEL-NEXT:    v_cvt_f16_f32_e32 v0, v0
; GFX6-GISEL-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f16x4_monotonic_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_add_f16_e32 v0, v0, v1
; GFX9-NEXT:    global_store_short v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f16x4_monotonic_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    v_mul_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    v_add_f16_e32 v0, v0, v1
; GFX10-NEXT:    global_store_short v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f16x4_monotonic_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    v_mul_f16_e32 v0.h, v1.l, v1.h
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX11-NEXT:    global_store_b16 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f16x4_monotonic_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    v_mul_f16_e32 v0.h, v1.l, v1.h
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_add_f16_e32 v0.l, v0.l, v0.h
; GFX12-NEXT:    global_store_b16 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
  %num1 = extractelement <4 x half> %a0, i32 0
  %num2 = extractelement <4 x half> %a0, i32 1
  %num3 = extractelement <4 x half> %a0, i32 2
  %num4 = extractelement <4 x half> %a0, i32 3
  %add = fadd half %num1, %num2
  %mul = fmul half %num3, %num4
  %res = fadd half %add, %mul
  store half %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
; GFX6-SDAG-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-SDAG-NEXT:    v_mad_u32_u24 v0, v1, v5, v0
; GFX6-SDAG-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
; GFX6-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-GISEL-NEXT:    v_mad_u32_u24 v0, v1, v5, v0
; GFX6-GISEL-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x4_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_mad_legacy_u16 v0, v1, v4, v0
; GFX9-NEXT:    global_store_short v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x4_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
; GFX10-NEXT:    v_add_nc_u16 v0, v0, v0 op_sel:[0,1,0]
; GFX10-NEXT:    v_mad_u16 v0, v1, v4, v0
; GFX10-NEXT:    global_store_short v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x4_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_mad_u16 v0.l, v1.l, v1.h, v0.l
; GFX11-NEXT:    global_store_b16 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x4_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_mad_u16 v0.l, v1.l, v1.h, v0.l
; GFX12-NEXT:    global_store_b16 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
  %num1 = extractelement <4 x i16> %a0, i32 0
  %num2 = extractelement <4 x i16> %a0, i32 1
  %num3 = extractelement <4 x i16> %a0, i32 2
  %num4 = extractelement <4 x i16> %a0, i32 3
  %add = add i16 %num1, %num2
  %mul = mul i16 %num3, %num4
  %res = add i16 %add, %mul
  store i16 %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x4_seq_cst_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_i16x4_seq_cst_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    buffer_wbinvl1
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
; GFX6-SDAG-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-SDAG-NEXT:    v_mad_u32_u24 v0, v1, v5, v0
; GFX6-SDAG-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_i16x4_seq_cst_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    buffer_wbinvl1
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
; GFX6-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-GISEL-NEXT:    v_mad_u32_u24 v0, v1, v5, v0
; GFX6-GISEL-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x4_seq_cst_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_mad_legacy_u16 v0, v1, v4, v0
; GFX9-NEXT:    global_store_short v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x4_seq_cst_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
; GFX10-NEXT:    v_add_nc_u16 v0, v0, v0 op_sel:[0,1,0]
; GFX10-NEXT:    v_mad_u16 v0, v1, v4, v0
; GFX10-NEXT:    global_store_short v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x4_seq_cst_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_mad_u16 v0.l, v1.l, v1.h, v0.l
; GFX11-NEXT:    global_store_b16 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x4_seq_cst_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_mad_u16 v0.l, v1.l, v1.h, v0.l
; GFX12-NEXT:    global_store_b16 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
  %num1 = extractelement <4 x i16> %a0, i32 0
  %num2 = extractelement <4 x i16> %a0, i32 1
  %num3 = extractelement <4 x i16> %a0, i32 2
  %num4 = extractelement <4 x i16> %a0, i32 3
  %add = add i16 %num1, %num2
  %mul = mul i16 %num3, %num4
  %res = add i16 %add, %mul
  store i16 %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x4_monotonic_wavefront(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_i16x4_monotonic_wavefront:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
; GFX6-SDAG-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-SDAG-NEXT:    v_mad_u32_u24 v0, v1, v5, v0
; GFX6-SDAG-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_i16x4_monotonic_wavefront:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
; GFX6-GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
; GFX6-GISEL-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-GISEL-NEXT:    v_mad_u32_u24 v0, v1, v5, v0
; GFX6-GISEL-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x4_monotonic_wavefront:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    v_mad_legacy_u16 v0, v1, v4, v0
; GFX9-NEXT:    global_store_short v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x4_monotonic_wavefront:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
; GFX10-NEXT:    v_add_nc_u16 v0, v0, v0 op_sel:[0,1,0]
; GFX10-NEXT:    v_mad_u16 v0, v1, v4, v0
; GFX10-NEXT:    global_store_short v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x4_monotonic_wavefront:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_mad_u16 v0.l, v1.l, v1.h, v0.l
; GFX11-NEXT:    global_store_b16 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x4_monotonic_wavefront:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_mad_u16 v0.l, v1.l, v1.h, v0.l
; GFX12-NEXT:    global_store_b16 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
  %num1 = extractelement <4 x i16> %a0, i32 0
  %num2 = extractelement <4 x i16> %a0, i32 1
  %num3 = extractelement <4 x i16> %a0, i32 2
  %num4 = extractelement <4 x i16> %a0, i32 3
  %add = add i16 %num1, %num2
  %mul = mul i16 %num3, %num4
  %res = add i16 %add, %mul
  store i16 %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_1(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 offset:1 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-SDAG-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 offset:1 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-GISEL-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off offset:1 glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX9-NEXT:    global_store_dword v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off offset:1 glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX10-NEXT:    global_store_dword v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off offset:1 glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX11-NEXT:    global_store_b32 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_1:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off offset:1 scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX12-NEXT:    global_store_b32 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 1
  %a0 = load atomic <2 x float>, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
  %num1 = extractelement <2 x float> %a0, i32 0
  %num2 = extractelement <2 x float> %a0, i32 1
  %res = fadd float %num1, %num2
  store float %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_f32x2_monotonic_agent_offset_max(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 offset:4095 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-SDAG-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 offset:4095 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX6-GISEL-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off offset:4095 glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX9-NEXT:    global_store_dword v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-SDAG-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
; GFX10-SDAG:       ; %bb.0:
; GFX10-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800, v0
; GFX10-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-SDAG-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off offset:2047 glc dlc
; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX10-SDAG-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX10-SDAG-NEXT:    global_store_dword v[2:3], v0, off
; GFX10-SDAG-NEXT:    s_endpgm
;
; GFX10-GISEL-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
; GFX10-GISEL:       ; %bb.0:
; GFX10-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 0xfff
; GFX10-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-GISEL-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc dlc
; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX10-GISEL-NEXT:    global_store_dword v[2:3], v0, off
; GFX10-GISEL-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off offset:4095 glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX11-NEXT:    global_store_b32 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f32x2_monotonic_agent_offset_max:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off offset:4095 scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_f32_e32 v0, v0, v1
; GFX12-NEXT:    global_store_b32 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 4095
  %a0 = load atomic <2 x float>, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
  %num1 = extractelement <2 x float> %a0, i32 0
  %num2 = extractelement <2 x float> %a0, i32 1
  %res = fadd float %num1, %num2
  store float %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_1(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:1 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
; GFX6-SDAG-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:1 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
; GFX6-GISEL-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:1 glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    global_store_short v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dword v0, v[0:1], off offset:1 glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_add_nc_u16 v0, v0, v0 op_sel:[0,1,0]
; GFX10-NEXT:    global_store_short v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b32 v0, v[0:1], off offset:1 glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX11-NEXT:    global_store_b16 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b32 v0, v[0:1], off offset:1 scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX12-NEXT:    global_store_b16 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 1
  %a = load atomic <2 x i16>, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
  %e0 = extractelement <2 x i16> %a, i32 0
  %e1 = extractelement <2 x i16> %a, i32 1
  %sum = add i16 %e0, %e1
  store i16 %sum, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:4095 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
; GFX6-SDAG-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:4095 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
; GFX6-GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
; GFX6-GISEL-NEXT:    buffer_store_short v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dword v0, v[0:1], off offset:4095 glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-NEXT:    global_store_short v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
; GFX10-SDAG:       ; %bb.0:
; GFX10-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, 0x800, v0
; GFX10-SDAG-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-SDAG-NEXT:    global_load_dword v0, v[0:1], off offset:2047 glc dlc
; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX10-SDAG-NEXT:    v_add_nc_u16 v0, v0, v0 op_sel:[0,1,0]
; GFX10-SDAG-NEXT:    global_store_short v[2:3], v0, off
; GFX10-SDAG-NEXT:    s_endpgm
;
; GFX10-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
; GFX10-GISEL:       ; %bb.0:
; GFX10-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 0xfff
; GFX10-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX10-GISEL-NEXT:    global_load_dword v0, v[0:1], off glc dlc
; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT:    v_add_nc_u16 v0, v0, v0 op_sel:[0,1,0]
; GFX10-GISEL-NEXT:    global_store_short v[2:3], v0, off
; GFX10-GISEL-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b32 v0, v[0:1], off offset:4095 glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX11-NEXT:    global_store_b16 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b32 v0, v[0:1], off offset:4095 scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
; GFX12-NEXT:    global_store_b16 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 4095
  %a = load atomic <2 x i16>, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
  %e0 = extractelement <2 x i16> %a, i32 0
  %e1 = extractelement <2 x i16> %a, i32 1
  %sum = add i16 %e0, %e1
  store i16 %sum, ptr addrspace(1) %out, align 4
  ret void
}
define amdgpu_cs void @atomic_load_i32x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_i32x2_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
; GFX6-SDAG-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_i32x2_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
; GFX6-GISEL-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_i32x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
; GFX9-NEXT:    global_store_dword v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_i32x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
; GFX10-NEXT:    global_store_dword v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_i32x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_nc_u32_e32 v0, v0, v1
; GFX11-NEXT:    global_store_b32 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i32x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_nc_u32_e32 v0, v0, v1
; GFX12-NEXT:    global_store_b32 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x i32>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
  %num1 = extractelement <2 x i32> %a0, i32 0
  %num2 = extractelement <2 x i32> %a0, i32 1
  %res = add i32 %num1, %num2
  store i32 %res, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_bf16x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_bf16x2_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_bf16x2_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_bf16x2_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dword v0, v[0:1], off glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    global_store_dword v[2:3], v0, off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_bf16x2_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dword v0, v[0:1], off glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    global_store_dword v[2:3], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_bf16x2_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b32 v0, v[0:1], off glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    global_store_b32 v[2:3], v0, off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_bf16x2_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_store_b32 v[2:3], v0, off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <2 x bfloat>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
  store <2 x bfloat> %a0, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_cs void @atomic_load_bf16x4_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX6-SDAG-LABEL: atomic_load_bf16x4_monotonic_agent:
; GFX6-SDAG:       ; %bb.0:
; GFX6-SDAG-NEXT:    s_mov_b32 s2, 0
; GFX6-SDAG-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-SDAG-NEXT:    s_mov_b32 s0, s2
; GFX6-SDAG-NEXT:    s_mov_b32 s1, s2
; GFX6-SDAG-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0)
; GFX6-SDAG-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-SDAG-NEXT:    s_endpgm
;
; GFX6-GISEL-LABEL: atomic_load_bf16x4_monotonic_agent:
; GFX6-GISEL:       ; %bb.0:
; GFX6-GISEL-NEXT:    s_mov_b32 s2, 0
; GFX6-GISEL-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-GISEL-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-GISEL-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc
; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0)
; GFX6-GISEL-NEXT:    buffer_store_dwordx2 v[0:1], v[2:3], s[0:3], 0 addr64
; GFX6-GISEL-NEXT:    s_endpgm
;
; GFX9-LABEL: atomic_load_bf16x4_monotonic_agent:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: atomic_load_bf16x4_monotonic_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off glc dlc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: atomic_load_bf16x4_monotonic_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_bf16x4_monotonic_agent:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    global_store_b64 v[2:3], v[0:1], off
; GFX12-NEXT:    s_endpgm
  %a0 = load atomic <4 x bfloat>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
  store <4 x bfloat> %a0, ptr addrspace(1) %out, align 8
  ret void
}

;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
; GFX11-GISEL: {{.*}}
; GFX11-SDAG: {{.*}}
; GFX12-GISEL: {{.*}}
; GFX12-SDAG: {{.*}}
; GFX6: {{.*}}
; GFX9-GISEL: {{.*}}
; GFX9-SDAG: {{.*}}