llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu7.04 < %s | FileCheck -check-prefix=GCN1 %s
; RUN: llc -mtriple=amdgpu8.02 < %s | FileCheck -check-prefix=GCN2 %s
; RUN: llc -mtriple=amdgpu12.00 < %s | FileCheck -check-prefix=GFX12 %s

define amdgpu_kernel void @atomic_add_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_add_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB0_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB0_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB0_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:  .LBB0_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB0_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB0_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB0_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:  .LBB0_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB0_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB0_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB0_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB0_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_add_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_add_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB1_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB1_3
; GCN1-NEXT:  .LBB1_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB1_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB1_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v5, vcc, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB1_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB1_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB1_3
; GCN2-NEXT:  .LBB1_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB1_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB1_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v5, vcc, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB1_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB1_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB1_3
; GFX12-NEXT:  .LBB1_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB1_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB1_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB1_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_add_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_add_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB2_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB2_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB2_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:  .LBB2_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xfc
; GCN2-NEXT:    s_add_u32 s0, s0, s6
; GCN2-NEXT:    s_addc_u32 s1, s1, s7
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB2_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB2_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB2_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:  .LBB2_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB2_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB2_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB2_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB2_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_add_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_add_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB3_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB3_3
; GCN1-NEXT:  .LBB3_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB3_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB3_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v5, vcc, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB3_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB3_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB3_3
; GCN2-NEXT:  .LBB3_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB3_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB3_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v5, vcc, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB3_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB3_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB3_3
; GFX12-NEXT:  .LBB3_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB3_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB3_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB3_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_add_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_add_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB4_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB4_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB4_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:  .LBB4_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB4_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB4_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB4_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:  .LBB4_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB4_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB4_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB4_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB4_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile add ptr %out, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_add_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_add_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB5_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB5_3
; GCN1-NEXT:  .LBB5_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB5_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB5_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v5, vcc, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB5_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB5_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB5_3
; GCN2-NEXT:  .LBB5_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB5_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB5_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v5, vcc, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB5_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB5_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB5_3
; GFX12-NEXT:  .LBB5_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB5_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB5_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB5_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile add ptr %out, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_add_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_add_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB6_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB6_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB6_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:  .LBB6_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB6_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB6_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB6_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:  .LBB6_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB6_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB6_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB6_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB6_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile add ptr %ptr, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_add_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_add_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB7_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB7_3
; GCN1-NEXT:  .LBB7_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB7_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB7_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v5, vcc, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB7_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB7_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB7_3
; GCN2-NEXT:  .LBB7_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB7_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB7_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v5, vcc, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB7_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB7_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB7_3
; GFX12-NEXT:  .LBB7_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB7_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB7_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB7_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile add ptr %ptr, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_and_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_and_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB8_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB8_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB8_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB8_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB8_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB8_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB8_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB8_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB8_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB8_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB8_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_and_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB8_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_and_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_and_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB9_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB9_3
; GCN1-NEXT:  .LBB9_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB9_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB9_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v4, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v5, s5, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB9_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB9_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB9_3
; GCN2-NEXT:  .LBB9_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB9_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB9_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v4, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v5, s5, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB9_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB9_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB9_3
; GFX12-NEXT:  .LBB9_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB9_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB9_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_and_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB9_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_and_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_and_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB10_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB10_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB10_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB10_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xfc
; GCN2-NEXT:    s_add_u32 s0, s0, s6
; GCN2-NEXT:    s_addc_u32 s1, s1, s7
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB10_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB10_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB10_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB10_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB10_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB10_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB10_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_and_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB10_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_and_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB11_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB11_3
; GCN1-NEXT:  .LBB11_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB11_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB11_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v4, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v5, s13, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB11_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB11_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB11_3
; GCN2-NEXT:  .LBB11_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB11_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB11_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v4, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v5, s13, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB11_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB11_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB11_3
; GFX12-NEXT:  .LBB11_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB11_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB11_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_and_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB11_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_and_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_and_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB12_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB12_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB12_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB12_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB12_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB12_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB12_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB12_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB12_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB12_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB12_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_and_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB12_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile and ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_and_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_and_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB13_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB13_3
; GCN1-NEXT:  .LBB13_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB13_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB13_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v4, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v5, s5, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB13_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB13_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB13_3
; GCN2-NEXT:  .LBB13_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB13_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB13_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v4, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v5, s5, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB13_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB13_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB13_3
; GFX12-NEXT:  .LBB13_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB13_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB13_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_and_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB13_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile and ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_and_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_and_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB14_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB14_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB14_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB14_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB14_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB14_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB14_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB14_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB14_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB14_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB14_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_and_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB14_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile and ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_and_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_and_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB15_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB15_3
; GCN1-NEXT:  .LBB15_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB15_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB15_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v4, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v5, s13, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB15_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB15_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB15_3
; GCN2-NEXT:  .LBB15_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB15_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB15_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v4, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v5, s13, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB15_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB15_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB15_3
; GFX12-NEXT:  .LBB15_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB15_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB15_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_and_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB15_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile and ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_sub_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_sub_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB16_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB16_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB16_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:  .LBB16_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB16_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB16_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB16_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:  .LBB16_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB16_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB16_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB16_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB16_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_sub_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_sub_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB17_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB17_3
; GCN1-NEXT:  .LBB17_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB17_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB17_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB17_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB17_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB17_3
; GCN2-NEXT:  .LBB17_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB17_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB17_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v5, vcc, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB17_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB17_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB17_3
; GFX12-NEXT:  .LBB17_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB17_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB17_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v3, null, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB17_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_sub_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_sub_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB18_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB18_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB18_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:  .LBB18_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xfc
; GCN2-NEXT:    s_add_u32 s0, s0, s6
; GCN2-NEXT:    s_addc_u32 s1, s1, s7
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB18_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB18_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB18_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:  .LBB18_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB18_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB18_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB18_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB18_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_sub_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB19_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB19_3
; GCN1-NEXT:  .LBB19_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB19_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB19_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v5, vcc, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB19_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB19_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB19_3
; GCN2-NEXT:  .LBB19_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB19_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB19_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v5, vcc, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB19_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB19_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB19_3
; GFX12-NEXT:  .LBB19_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB19_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB19_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v3, null, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB19_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_sub_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_sub_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB20_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB20_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB20_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:  .LBB20_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB20_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB20_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB20_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:  .LBB20_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB20_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB20_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB20_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB20_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile sub ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_sub_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_sub_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB21_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB21_3
; GCN1-NEXT:  .LBB21_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB21_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB21_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB21_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB21_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB21_3
; GCN2-NEXT:  .LBB21_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB21_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB21_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v5, vcc, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB21_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB21_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB21_3
; GFX12-NEXT:  .LBB21_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB21_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB21_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v3, null, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB21_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile sub ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_sub_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_sub_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB22_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB22_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB22_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:  .LBB22_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB22_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB22_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB22_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:  .LBB22_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB22_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB22_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB22_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB22_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile sub ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_sub_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_sub_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB23_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB23_3
; GCN1-NEXT:  .LBB23_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB23_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB23_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v5, vcc, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB23_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB23_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB23_3
; GCN2-NEXT:  .LBB23_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB23_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB23_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v5, vcc, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB23_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB23_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB23_3
; GFX12-NEXT:  .LBB23_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB23_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB23_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v3, null, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB23_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile sub ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_max_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_max_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB24_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB24_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB24_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB24_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB24_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB24_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB24_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB24_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB24_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB24_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB24_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB24_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_max_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_max_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB25_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB25_3
; GCN1-NEXT:  .LBB25_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB25_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB25_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB25_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB25_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB25_3
; GCN2-NEXT:  .LBB25_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB25_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB25_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB25_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB25_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB25_3
; GFX12-NEXT:  .LBB25_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB25_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB25_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB25_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_max_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB26_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB26_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB26_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB26_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xfc
; GCN2-NEXT:    s_add_u32 s0, s0, s6
; GCN2-NEXT:    s_addc_u32 s1, s1, s7
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB26_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB26_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB26_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB26_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB26_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB26_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB26_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB26_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_max_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB27_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB27_3
; GCN1-NEXT:  .LBB27_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB27_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB27_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB27_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB27_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB27_3
; GCN2-NEXT:  .LBB27_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB27_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB27_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB27_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB27_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB27_3
; GFX12-NEXT:  .LBB27_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB27_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB27_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB27_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_max_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_max_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB28_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB28_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB28_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB28_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB28_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB28_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB28_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB28_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB28_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB28_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB28_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB28_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile max ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_max_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_max_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB29_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB29_3
; GCN1-NEXT:  .LBB29_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB29_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB29_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB29_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB29_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB29_3
; GCN2-NEXT:  .LBB29_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB29_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB29_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB29_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB29_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB29_3
; GFX12-NEXT:  .LBB29_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB29_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB29_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB29_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile max ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_max_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB30_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB30_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB30_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB30_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB30_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB30_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB30_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB30_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB30_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB30_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB30_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB30_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile max ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_max_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB31_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB31_3
; GCN1-NEXT:  .LBB31_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB31_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB31_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB31_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB31_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB31_3
; GCN2-NEXT:  .LBB31_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB31_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB31_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB31_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB31_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB31_3
; GFX12-NEXT:  .LBB31_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB31_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB31_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB31_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile max ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umax_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_umax_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB32_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB32_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB32_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB32_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB32_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB32_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB32_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB32_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB32_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB32_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB32_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB32_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_umax_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_umax_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB33_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB33_3
; GCN1-NEXT:  .LBB33_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB33_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB33_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB33_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB33_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB33_3
; GCN2-NEXT:  .LBB33_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB33_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB33_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB33_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB33_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB33_3
; GFX12-NEXT:  .LBB33_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB33_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB33_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB33_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umax_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB34_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB34_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB34_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB34_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xfc
; GCN2-NEXT:    s_add_u32 s0, s0, s6
; GCN2-NEXT:    s_addc_u32 s1, s1, s7
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB34_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB34_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB34_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB34_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB34_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB34_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB34_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB34_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umax_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB35_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB35_3
; GCN1-NEXT:  .LBB35_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB35_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB35_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB35_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB35_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB35_3
; GCN2-NEXT:  .LBB35_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB35_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB35_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB35_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB35_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB35_3
; GFX12-NEXT:  .LBB35_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB35_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB35_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB35_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umax_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_umax_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB36_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB36_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB36_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB36_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB36_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB36_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB36_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB36_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB36_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB36_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB36_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB36_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile umax ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_umax_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_umax_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB37_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB37_3
; GCN1-NEXT:  .LBB37_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB37_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB37_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB37_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB37_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB37_3
; GCN2-NEXT:  .LBB37_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB37_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB37_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB37_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB37_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB37_3
; GFX12-NEXT:  .LBB37_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB37_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB37_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB37_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile umax ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umax_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umax_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB38_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB38_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB38_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB38_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB38_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB38_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB38_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB38_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB38_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB38_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB38_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB38_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile umax ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umax_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB39_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB39_3
; GCN1-NEXT:  .LBB39_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB39_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB39_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB39_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB39_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB39_3
; GCN2-NEXT:  .LBB39_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB39_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB39_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB39_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB39_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB39_3
; GFX12-NEXT:  .LBB39_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB39_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB39_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB39_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile umax ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_min_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_min_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB40_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB40_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB40_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB40_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB40_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB40_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB40_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB40_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB40_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB40_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB40_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB40_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_min_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_min_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB41_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB41_3
; GCN1-NEXT:  .LBB41_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB41_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB41_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB41_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB41_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB41_3
; GCN2-NEXT:  .LBB41_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB41_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB41_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB41_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB41_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB41_3
; GFX12-NEXT:  .LBB41_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB41_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB41_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB41_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_min_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB42_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB42_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB42_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB42_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xfc
; GCN2-NEXT:    s_add_u32 s0, s0, s6
; GCN2-NEXT:    s_addc_u32 s1, s1, s7
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB42_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB42_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB42_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB42_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB42_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB42_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB42_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB42_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_min_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB43_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB43_3
; GCN1-NEXT:  .LBB43_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB43_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB43_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB43_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB43_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB43_3
; GCN2-NEXT:  .LBB43_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB43_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB43_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB43_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB43_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB43_3
; GFX12-NEXT:  .LBB43_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB43_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB43_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB43_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_min_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB44_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB44_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB44_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB44_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB44_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB44_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB44_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB44_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB44_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB44_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB44_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB44_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile min ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_min_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_min_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB45_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB45_3
; GCN1-NEXT:  .LBB45_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB45_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB45_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB45_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB45_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB45_3
; GCN2-NEXT:  .LBB45_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB45_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB45_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB45_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB45_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB45_3
; GFX12-NEXT:  .LBB45_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB45_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB45_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB45_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile min ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_min_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_min_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB46_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB46_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB46_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB46_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB46_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB46_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB46_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB46_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB46_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB46_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB46_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB46_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile min ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_min_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB47_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB47_3
; GCN1-NEXT:  .LBB47_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB47_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB47_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB47_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB47_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB47_3
; GCN2-NEXT:  .LBB47_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB47_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB47_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB47_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB47_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB47_3
; GFX12-NEXT:  .LBB47_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB47_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB47_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB47_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile min ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umin_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_umin_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB48_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB48_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB48_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB48_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB48_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB48_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB48_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB48_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB48_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB48_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB48_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB48_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_umin_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_umin_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB49_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB49_3
; GCN1-NEXT:  .LBB49_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB49_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB49_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB49_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB49_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB49_3
; GCN2-NEXT:  .LBB49_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB49_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB49_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB49_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB49_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB49_3
; GFX12-NEXT:  .LBB49_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB49_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB49_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB49_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umin_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umin_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB50_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB50_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB50_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB50_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xfc
; GCN2-NEXT:    s_add_u32 s0, s0, s6
; GCN2-NEXT:    s_addc_u32 s1, s1, s7
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB50_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB50_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB50_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB50_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB50_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB50_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB50_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB50_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umin_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB51_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB51_3
; GCN1-NEXT:  .LBB51_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB51_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB51_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB51_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB51_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB51_3
; GCN2-NEXT:  .LBB51_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB51_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB51_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB51_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB51_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB51_3
; GFX12-NEXT:  .LBB51_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB51_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB51_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB51_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umin_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_umin_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB52_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB52_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB52_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB52_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB52_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB52_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB52_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB52_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB52_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB52_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB52_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB52_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile umin ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_umin_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_umin_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB53_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB53_3
; GCN1-NEXT:  .LBB53_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB53_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB53_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB53_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB53_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB53_3
; GCN2-NEXT:  .LBB53_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB53_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB53_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB53_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB53_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB53_3
; GFX12-NEXT:  .LBB53_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB53_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB53_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB53_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile umin ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umin_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umin_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB54_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB54_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB54_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB54_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB54_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB54_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB54_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB54_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB54_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:  .LBB54_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB54_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB54_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile umin ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_umin_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umin_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB55_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB55_3
; GCN1-NEXT:  .LBB55_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB55_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB55_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB55_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB55_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB55_3
; GCN2-NEXT:  .LBB55_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB55_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB55_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB55_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB55_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_branch .LBB55_3
; GFX12-NEXT:  .LBB55_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB55_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB55_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB55_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile umin ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_or_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_or_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB56_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB56_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB56_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB56_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB56_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB56_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB56_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB56_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB56_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB56_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB56_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_or_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB56_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_or_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_or_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB57_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB57_3
; GCN1-NEXT:  .LBB57_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB57_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB57_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v4, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v5, s5, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB57_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB57_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB57_3
; GCN2-NEXT:  .LBB57_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB57_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB57_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v4, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v5, s5, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB57_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB57_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB57_3
; GFX12-NEXT:  .LBB57_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB57_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB57_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_or_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB57_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_or_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_or_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB58_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB58_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB58_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB58_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xfc
; GCN2-NEXT:    s_add_u32 s0, s0, s6
; GCN2-NEXT:    s_addc_u32 s1, s1, s7
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB58_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB58_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB58_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB58_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB58_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB58_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB58_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_or_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB58_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_or_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB59_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB59_3
; GCN1-NEXT:  .LBB59_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB59_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB59_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v4, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v5, s13, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB59_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB59_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB59_3
; GCN2-NEXT:  .LBB59_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB59_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB59_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v4, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v5, s13, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB59_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB59_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB59_3
; GFX12-NEXT:  .LBB59_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB59_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB59_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_or_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB59_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_or_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_or_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB60_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB60_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB60_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB60_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB60_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB60_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB60_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB60_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB60_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB60_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB60_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_or_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB60_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile or ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_or_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_or_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB61_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB61_3
; GCN1-NEXT:  .LBB61_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB61_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB61_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v4, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v5, s5, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB61_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB61_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB61_3
; GCN2-NEXT:  .LBB61_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB61_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB61_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v4, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v5, s5, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB61_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB61_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB61_3
; GFX12-NEXT:  .LBB61_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB61_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB61_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_or_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB61_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile or ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_or_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_or_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB62_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB62_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB62_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB62_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB62_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB62_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB62_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB62_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB62_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB62_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB62_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_or_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB62_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile or ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_or_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_or_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB63_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB63_3
; GCN1-NEXT:  .LBB63_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB63_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB63_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v4, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v5, s13, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB63_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB63_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB63_3
; GCN2-NEXT:  .LBB63_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB63_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB63_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v4, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v5, s13, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB63_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB63_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB63_3
; GFX12-NEXT:  .LBB63_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB63_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB63_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_or_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB63_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile or ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_xchg_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB64_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB64_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB64_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v0, s3
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB64_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB64_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB64_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB64_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v0, s3
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB64_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB64_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB64_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB64_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB64_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xchg_f64_offset(ptr %out, double %in) {
; GCN1-LABEL: atomic_xchg_f64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB65_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB65_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB65_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v0, s3
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB65_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_f64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB65_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB65_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB65_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v0, s3
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB65_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_f64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB65_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB65_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB65_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB65_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds double, ptr %out, i64 4
  %tmp0 = atomicrmw volatile xchg ptr %gep, double %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xchg_pointer_offset(ptr %out, ptr %in) {
; GCN1-LABEL: atomic_xchg_pointer_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB66_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB66_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB66_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v0, s3
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB66_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_pointer_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB66_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB66_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB66_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v0, s3
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB66_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_pointer_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB66_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB66_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB66_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB66_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds ptr, ptr %out, i32 4
  %val = atomicrmw volatile xchg ptr %gep, ptr %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_xchg_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB67_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB67_3
; GCN1-NEXT:  .LBB67_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB67_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB67_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:    buffer_store_dword v2, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB67_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(2)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB67_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB67_3
; GCN2-NEXT:  .LBB67_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB67_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB67_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s4
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:    buffer_store_dword v2, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB67_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(2)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB67_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB67_3
; GFX12-NEXT:  .LBB67_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB67_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB67_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB67_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xchg_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB68_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB68_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB68_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v0, s3
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB68_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xfc
; GCN2-NEXT:    s_add_u32 s0, s0, s6
; GCN2-NEXT:    s_addc_u32 s1, s1, s7
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB68_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB68_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB68_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v0, s3
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB68_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB68_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB68_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB68_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB68_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xchg_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB69_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB69_3
; GCN1-NEXT:  .LBB69_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB69_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB69_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s13
; GCN1-NEXT:    buffer_store_dword v2, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB69_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(2)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB69_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB69_3
; GCN2-NEXT:  .LBB69_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB69_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB69_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s12
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s13
; GCN2-NEXT:    buffer_store_dword v2, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB69_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(2)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB69_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB69_3
; GFX12-NEXT:  .LBB69_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB69_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB69_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB69_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_xchg_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB70_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB70_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB70_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v0, s3
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB70_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB70_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB70_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB70_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v0, s3
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB70_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB70_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB70_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB70_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB70_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile xchg ptr %out, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_xchg_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB71_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB71_3
; GCN1-NEXT:  .LBB71_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB71_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB71_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:    buffer_store_dword v2, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB71_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(2)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB71_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB71_3
; GCN2-NEXT:  .LBB71_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB71_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB71_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s4
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:    buffer_store_dword v2, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB71_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(2)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB71_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB71_3
; GFX12-NEXT:  .LBB71_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB71_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB71_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB71_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile xchg ptr %out, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xchg_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB72_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB72_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB72_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v0, s3
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB72_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB72_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB72_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB72_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v0, s3
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB72_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB72_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB72_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB72_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB72_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile xchg ptr %ptr, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xchg_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB73_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB73_3
; GCN1-NEXT:  .LBB73_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB73_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB73_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s13
; GCN1-NEXT:    buffer_store_dword v2, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB73_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(2)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB73_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB73_3
; GCN2-NEXT:  .LBB73_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB73_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB73_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s12
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s13
; GCN2-NEXT:    buffer_store_dword v2, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB73_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(2)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB73_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB73_3
; GFX12-NEXT:  .LBB73_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB73_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB73_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB73_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile xchg ptr %ptr, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xor_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_xor_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB74_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB74_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB74_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB74_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB74_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB74_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB74_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB74_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB74_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB74_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB74_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_xor_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB74_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_xor_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_xor_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB75_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB75_3
; GCN1-NEXT:  .LBB75_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB75_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB75_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v4, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v5, s5, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB75_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB75_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB75_3
; GCN2-NEXT:  .LBB75_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB75_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB75_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v4, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v5, s5, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB75_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB75_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB75_3
; GFX12-NEXT:  .LBB75_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB75_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB75_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_xor_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB75_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xor_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xor_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB76_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB76_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB76_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB76_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xfc
; GCN2-NEXT:    s_add_u32 s0, s0, s6
; GCN2-NEXT:    s_addc_u32 s1, s1, s7
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB76_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB76_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB76_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB76_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB76_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB76_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB76_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_xor_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB76_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xor_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB77_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB77_3
; GCN1-NEXT:  .LBB77_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB77_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB77_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v4, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v5, s13, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB77_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB77_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB77_3
; GCN2-NEXT:  .LBB77_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB77_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB77_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v4, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v5, s13, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB77_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB77_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB77_3
; GFX12-NEXT:  .LBB77_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB77_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB77_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_xor_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB77_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xor_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_xor_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB78_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB78_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB78_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB78_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB78_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB78_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB78_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB78_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB78_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB78_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB78_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_xor_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB78_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile xor ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_xor_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_xor_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB79_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB79_3
; GCN1-NEXT:  .LBB79_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB79_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB79_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v4, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v5, s5, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB79_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB79_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB79_3
; GCN2-NEXT:  .LBB79_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB79_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB79_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v4, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v5, s5, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB79_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB79_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB79_3
; GFX12-NEXT:  .LBB79_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB79_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB79_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_xor_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB79_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile xor ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xor_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xor_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB80_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB80_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB80_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:  .LBB80_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB80_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB80_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB80_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:  .LBB80_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB80_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB80_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB80_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_xor_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB80_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile xor ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_xor_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xor_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB81_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB81_3
; GCN1-NEXT:  .LBB81_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB81_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB81_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v4, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v5, s13, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB81_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB81_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB81_3
; GCN2-NEXT:  .LBB81_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB81_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB81_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v4, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v5, s13, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB81_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB81_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB81_3
; GFX12-NEXT:  .LBB81_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB81_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB81_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_xor_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB81_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile xor ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_load_i64_offset(ptr %in, ptr %out) {
; GCN1-LABEL: atomic_load_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %in, i64 4
  %val = load atomic i64, ptr %gep  seq_cst, align 8
  store i64 %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_load_i64(ptr %in, ptr %out) {
; GCN1-LABEL: atomic_load_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %val = load atomic i64, ptr %in syncscope("agent") seq_cst, align 8
  store i64 %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_load_i64_addr64_offset(ptr %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_load_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %in, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %val = load atomic i64, ptr %gep seq_cst, align 8
  store i64 %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_load_i64_addr64(ptr %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_load_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %in, i64 %index
  %val = load atomic i64, ptr %ptr seq_cst, align 8
  store i64 %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_store_i64_offset(i64 %in, ptr %out) {
; GCN1-LABEL: atomic_store_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_u32 s0, s2, 32
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    s_addc_u32 s1, s3, 0
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_u32 s0, s2, 32
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    s_addc_u32 s1, s3, 0
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  store atomic i64 %in, ptr %gep  seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_store_i64(i64 %in, ptr %out) {
; GCN1-LABEL: atomic_store_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  store atomic i64 %in, ptr %out seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_store_i64_addr64_offset(i64 %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_store_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN1-NEXT:    s_add_u32 s0, s2, s0
; GCN1-NEXT:    s_addc_u32 s1, s3, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN2-NEXT:    s_add_u32 s0, s2, s0
; GCN2-NEXT:    s_addc_u32 s1, s3, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  store atomic i64 %in, ptr %gep seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_store_i64_addr64(i64 %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_store_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN1-NEXT:    s_add_u32 s0, s2, s0
; GCN1-NEXT:    s_addc_u32 s1, s3, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN2-NEXT:    s_add_u32 s0, s2, s0
; GCN2-NEXT:    s_addc_u32 s1, s3, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  store atomic i64 %in, ptr %ptr seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_offset(ptr %out, i64 %in, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB90_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s3
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    v_mov_b32_e32 v5, s1
; GCN1-NEXT:    v_mov_b32_e32 v4, s0
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:  .LBB90_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB90_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB90_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB90_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s3
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    v_mov_b32_e32 v5, s1
; GCN2-NEXT:    v_mov_b32_e32 v4, s0
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:  .LBB90_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB90_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB90_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB90_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB90_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB90_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB90_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_soffset(ptr %out, i64 %in, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_soffset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 0x11940
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB91_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s3
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    v_mov_b32_e32 v5, s1
; GCN1-NEXT:    v_mov_b32_e32 v4, s0
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:  .LBB91_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB91_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB91_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_soffset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 0x11940
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB91_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s3
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    v_mov_b32_e32 v5, s1
; GCN2-NEXT:    v_mov_b32_e32 v4, s0
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:  .LBB91_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB91_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB91_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_soffset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 0x11940
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB91_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB91_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB91_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB91_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 9000
  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_ret_offset(ptr %out, ptr %out2, i64 %in, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s8, 32
; GCN1-NEXT:    s_addc_u32 s1, s9, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB92_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s13
; GCN1-NEXT:    v_mov_b32_e32 v2, s14
; GCN1-NEXT:    v_mov_b32_e32 v3, s15
; GCN1-NEXT:    v_mov_b32_e32 v5, s1
; GCN1-NEXT:    v_mov_b32_e32 v4, s0
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB92_3
; GCN1-NEXT:  .LBB92_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB92_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB92_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB92_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s8, 32
; GCN2-NEXT:    s_addc_u32 s1, s9, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB92_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s13
; GCN2-NEXT:    v_mov_b32_e32 v2, s14
; GCN2-NEXT:    v_mov_b32_e32 v3, s15
; GCN2-NEXT:    v_mov_b32_e32 v5, s1
; GCN2-NEXT:    v_mov_b32_e32 v4, s0
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB92_3
; GCN2-NEXT:  .LBB92_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB92_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB92_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB92_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s8, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s8, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB92_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
; GFX12-NEXT:    v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
; GFX12-NEXT:    s_mov_b32 s8, 0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB92_3
; GFX12-NEXT:  .LBB92_2:
; GFX12-NEXT:    s_mov_b32 s8, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB92_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s8, s8, exec_lo
; GFX12-NEXT:    s_cselect_b32 s8, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s8, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB92_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[6:7], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v1, s5, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v0, s4, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB92_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  %extract0 = extractvalue { i64, i1 } %val, 0
  store i64 %extract0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_addr64_offset(ptr %out, i64 %in, i64 %index, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[12:13], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB93_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s10
; GCN1-NEXT:    v_mov_b32_e32 v1, s11
; GCN1-NEXT:    v_mov_b32_e32 v2, s14
; GCN1-NEXT:    v_mov_b32_e32 v3, s15
; GCN1-NEXT:    v_mov_b32_e32 v5, s1
; GCN1-NEXT:    v_mov_b32_e32 v4, s0
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:  .LBB93_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB93_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s10
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB93_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[12:13], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB93_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s10
; GCN2-NEXT:    v_mov_b32_e32 v1, s11
; GCN2-NEXT:    v_mov_b32_e32 v2, s14
; GCN2-NEXT:    v_mov_b32_e32 v3, s15
; GCN2-NEXT:    v_mov_b32_e32 v5, s1
; GCN2-NEXT:    v_mov_b32_e32 v4, s0
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:  .LBB93_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB93_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s10
; GCN2-NEXT:    v_mov_b32_e32 v4, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB93_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[4:5], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB93_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB93_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB93_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[6:7], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB93_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x11
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x43
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[2:3], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s2, s8, s2
; GCN1-NEXT:    s_addc_u32 s3, s9, s3
; GCN1-NEXT:    s_add_u32 s2, s2, 32
; GCN1-NEXT:    s_addc_u32 s3, s3, 0
; GCN1-NEXT:    s_cmp_eq_u32 s3, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB94_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s13
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v5, s3
; GCN1-NEXT:    v_mov_b32_e32 v4, s2
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:    s_branch .LBB94_3
; GCN1-NEXT:  .LBB94_2:
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB94_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB94_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[2:3], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s2, s2, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    s_add_i32 s2, s2, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s2
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[0:1], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB94_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x44
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0x10c
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[2:3], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s2, s8, s2
; GCN2-NEXT:    s_addc_u32 s3, s9, s3
; GCN2-NEXT:    s_add_u32 s2, s2, 32
; GCN2-NEXT:    s_addc_u32 s3, s3, 0
; GCN2-NEXT:    s_cmp_eq_u32 s3, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB94_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s13
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v5, s3
; GCN2-NEXT:    v_mov_b32_e32 v4, s2
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:    s_branch .LBB94_3
; GCN2-NEXT:  .LBB94_2:
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB94_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB94_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[2:3], 0
; GCN2-NEXT:    s_cselect_b32 s2, s2, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    s_add_i32 s2, s2, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s2
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[0:1], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB94_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b256 s[8:15], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x44
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[2:3], s[14:15], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[8:9], s[2:3]
; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s3, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB94_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX12-NEXT:    v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX12-NEXT:    v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB94_3
; GFX12-NEXT:  .LBB94_2:
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB94_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB94_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[2:3], 0
; GFX12-NEXT:    s_cselect_b32 s2, s2, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s2
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[0:1], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v1, s13, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v0, s12, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s2
; GFX12-NEXT:  .LBB94_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v3, s11
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  %extract0 = extractvalue { i64, i1 } %val, 0
  store i64 %extract0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64(ptr %out, i64 %in, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB95_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s3
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    v_mov_b32_e32 v5, s1
; GCN1-NEXT:    v_mov_b32_e32 v4, s0
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:  .LBB95_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB95_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB95_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB95_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s3
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    v_mov_b32_e32 v5, s1
; GCN2-NEXT:    v_mov_b32_e32 v4, s0
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:  .LBB95_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB95_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB95_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB95_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB95_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB95_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB95_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %val = cmpxchg volatile ptr %out, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_ret(ptr %out, ptr %out2, i64 %in, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s0, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s9, s0
; GCN1-NEXT:    s_cselect_b32 s0, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s0, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB96_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s13
; GCN1-NEXT:    v_mov_b32_e32 v2, s14
; GCN1-NEXT:    v_mov_b32_e32 v3, s15
; GCN1-NEXT:    v_mov_b32_e32 v4, s8
; GCN1-NEXT:    v_mov_b32_e32 v5, s9
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[0:1], 0
; GCN1-NEXT:    s_branch .LBB96_3
; GCN1-NEXT:  .LBB96_2:
; GCN1-NEXT:    s_mov_b64 s[0:1], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB96_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[0:1], s[0:1], exec
; GCN1-NEXT:    s_cselect_b32 s0, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s0, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB96_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[0:1], s[8:9], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[0:1], s[0:1], exec
; GCN1-NEXT:    s_cselect_b32 s0, s8, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB96_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s0, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s9, s0
; GCN2-NEXT:    s_cselect_b32 s0, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s0, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB96_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s13
; GCN2-NEXT:    v_mov_b32_e32 v2, s14
; GCN2-NEXT:    v_mov_b32_e32 v3, s15
; GCN2-NEXT:    v_mov_b32_e32 v4, s8
; GCN2-NEXT:    v_mov_b32_e32 v5, s9
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[0:1], 0
; GCN2-NEXT:    s_branch .LBB96_3
; GCN2-NEXT:  .LBB96_2:
; GCN2-NEXT:    s_mov_b64 s[0:1], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB96_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[0:1], s[0:1], exec
; GCN2-NEXT:    s_cselect_b32 s0, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s0, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB96_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[8:9], 0
; GCN2-NEXT:    s_cselect_b32 s0, s8, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB96_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s8, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s8, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB96_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
; GFX12-NEXT:    v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
; GFX12-NEXT:    s_mov_b32 s8, 0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB96_3
; GFX12-NEXT:  .LBB96_2:
; GFX12-NEXT:    s_mov_b32 s8, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB96_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s8, s8, exec_lo
; GFX12-NEXT:    s_cselect_b32 s8, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s8, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB96_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[6:7], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v1, s5, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v0, s4, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB96_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %val = cmpxchg volatile ptr %out, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  %extract0 = extractvalue { i64, i1 } %val, 0
  store i64 %extract0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_addr64(ptr %out, i64 %in, i64 %index, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[12:13], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB97_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s10
; GCN1-NEXT:    v_mov_b32_e32 v1, s11
; GCN1-NEXT:    v_mov_b32_e32 v2, s14
; GCN1-NEXT:    v_mov_b32_e32 v3, s15
; GCN1-NEXT:    v_mov_b32_e32 v5, s1
; GCN1-NEXT:    v_mov_b32_e32 v4, s0
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:  .LBB97_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB97_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s10
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB97_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[12:13], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB97_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s10
; GCN2-NEXT:    v_mov_b32_e32 v1, s11
; GCN2-NEXT:    v_mov_b32_e32 v2, s14
; GCN2-NEXT:    v_mov_b32_e32 v3, s15
; GCN2-NEXT:    v_mov_b32_e32 v5, s1
; GCN2-NEXT:    v_mov_b32_e32 v4, s0
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:  .LBB97_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB97_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s10
; GCN2-NEXT:    v_mov_b32_e32 v4, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB97_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[4:5], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB97_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB97_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB97_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[6:7], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB97_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %val = cmpxchg volatile ptr %ptr, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x43
; GCN1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x11
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[2:3], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s2, s8, s2
; GCN1-NEXT:    s_addc_u32 s3, s9, s3
; GCN1-NEXT:    s_cmp_eq_u32 s3, s6
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB98_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s13
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v5, s3
; GCN1-NEXT:    v_mov_b32_e32 v4, s2
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:    s_branch .LBB98_3
; GCN1-NEXT:  .LBB98_2:
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB98_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB98_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[2:3], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s2, s2, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    s_add_i32 s2, s2, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s2
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[0:1], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB98_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0x10c
; GCN2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x44
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[2:3], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s2, s8, s2
; GCN2-NEXT:    s_addc_u32 s3, s9, s3
; GCN2-NEXT:    s_cmp_eq_u32 s3, s6
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB98_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s13
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v5, s3
; GCN2-NEXT:    v_mov_b32_e32 v4, s2
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:    s_branch .LBB98_3
; GCN2-NEXT:  .LBB98_2:
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB98_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB98_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[2:3], 0
; GCN2-NEXT:    s_cselect_b32 s2, s2, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    s_add_i32 s2, s2, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s2
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[0:1], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB98_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b256 s[8:15], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x44
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[2:3], s[14:15], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[8:9], s[2:3]
; GFX12-NEXT:    s_cmp_eq_u32 s3, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB98_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX12-NEXT:    v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX12-NEXT:    v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB98_3
; GFX12-NEXT:  .LBB98_2:
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB98_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB98_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[2:3], 0
; GFX12-NEXT:    s_cselect_b32 s2, s2, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s2
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[0:1], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v1, s13, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v0, s12, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s2
; GFX12-NEXT:  .LBB98_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v3, s11
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %val = cmpxchg volatile ptr %ptr, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  %extract0 = extractvalue { i64, i1 } %val, 0
  store i64 %extract0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_load_f64_offset(ptr %in, ptr %out) {
; GCN1-LABEL: atomic_load_f64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_f64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds double, ptr %in, i64 4
  %val = load atomic double, ptr %gep  seq_cst, align 8
  store double %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_load_f64(ptr %in, ptr %out) {
; GCN1-LABEL: atomic_load_f64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_f64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %val = load atomic double, ptr %in syncscope("agent") seq_cst, align 8
  store double %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_load_f64_addr64_offset(ptr %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_load_f64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_f64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds double, ptr %in, i64 %index
  %gep = getelementptr inbounds double, ptr %ptr, i64 4
  %val = load atomic double, ptr %gep seq_cst, align 8
  store double %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_load_f64_addr64(ptr %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_load_f64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_f64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds double, ptr %in, i64 %index
  %val = load atomic double, ptr %ptr seq_cst, align 8
  store double %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_store_f64_offset(double %in, ptr %out) {
; GCN1-LABEL: atomic_store_f64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_u32 s0, s2, 32
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    s_addc_u32 s1, s3, 0
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_f64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_u32 s0, s2, 32
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    s_addc_u32 s1, s3, 0
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds double, ptr %out, i64 4
  store atomic double %in, ptr %gep  seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_store_f64(double %in, ptr %out) {
; GCN1-LABEL: atomic_store_f64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_f64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  store atomic double %in, ptr %out seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_store_f64_addr64_offset(double %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_store_f64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN1-NEXT:    s_add_u32 s0, s2, s0
; GCN1-NEXT:    s_addc_u32 s1, s3, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_f64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN2-NEXT:    s_add_u32 s0, s2, s0
; GCN2-NEXT:    s_addc_u32 s1, s3, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds double, ptr %out, i64 %index
  %gep = getelementptr inbounds double, ptr %ptr, i64 4
  store atomic double %in, ptr %gep seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_store_f64_addr64(double %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_store_f64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN1-NEXT:    s_add_u32 s0, s2, s0
; GCN1-NEXT:    s_addc_u32 s1, s3, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_f64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN2-NEXT:    s_add_u32 s0, s2, s0
; GCN2-NEXT:    s_addc_u32 s1, s3, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_storecnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds double, ptr %out, i64 %index
  store atomic double %in, ptr %ptr seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_inc_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB107_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB107_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB107_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v1, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v0, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB107_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB107_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB107_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB107_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v1, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v0, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB107_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB107_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB107_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB107_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT:    v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB107_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_inc_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_inc_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB108_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB108_3
; GCN1-NEXT:  .LBB108_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB108_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB108_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB108_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB108_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB108_3
; GCN2-NEXT:  .LBB108_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB108_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB108_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB108_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB108_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB108_3
; GFX12-NEXT:  .LBB108_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB108_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB108_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT:    v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB108_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_inc_i64_incr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB109_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB109_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB109_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v1, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v0, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB109_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64_incr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xfc
; GCN2-NEXT:    s_add_u32 s0, s0, s6
; GCN2-NEXT:    s_addc_u32 s1, s1, s7
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB109_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB109_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB109_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v1, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v0, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB109_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_incr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB109_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB109_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB109_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT:    v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB109_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_inc_i64_ret_incr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_inc_i64_ret_incr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB110_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB110_3
; GCN1-NEXT:  .LBB110_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB110_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB110_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB110_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64_ret_incr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB110_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB110_3
; GCN2-NEXT:  .LBB110_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB110_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB110_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB110_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_ret_incr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB110_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB110_3
; GFX12-NEXT:  .LBB110_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB110_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB110_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT:    v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB110_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_inc_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_inc_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB111_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB111_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB111_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v1, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v0, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB111_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB111_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB111_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB111_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v1, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v0, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB111_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB111_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB111_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB111_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT:    v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB111_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile uinc_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_inc_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_inc_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB112_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB112_3
; GCN1-NEXT:  .LBB112_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB112_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB112_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB112_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB112_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB112_3
; GCN2-NEXT:  .LBB112_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB112_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB112_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB112_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB112_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB112_3
; GFX12-NEXT:  .LBB112_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB112_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB112_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT:    v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB112_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile uinc_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_inc_i64_incr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB113_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB113_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB113_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v1, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v0, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB113_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64_incr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB113_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB113_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB113_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v1, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v0, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB113_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_incr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB113_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB113_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB113_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT:    v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:  .LBB113_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile uinc_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_inc_i64_ret_incr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_inc_i64_ret_incr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB114_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB114_3
; GCN1-NEXT:  .LBB114_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB114_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB114_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB114_5: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64_ret_incr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB114_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB114_3
; GCN2-NEXT:  .LBB114_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB114_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB114_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB114_5: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_ret_incr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB114_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB114_3
; GFX12-NEXT:  .LBB114_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB114_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB114_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT:    v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB114_5: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile uinc_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_dec_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB115_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB115_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB115_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_subrev_i32_e64 v0, s[0:1], 1, v0
; GCN1-NEXT:    v_subbrev_u32_e64 v1, s[0:1], 0, v1, s[0:1]
; GCN1-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB115_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB115_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB115_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB115_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_subrev_u32_e64 v0, s[0:1], 1, v0
; GCN2-NEXT:    v_subbrev_u32_e64 v1, s[0:1], 0, v1, s[0:1]
; GCN2-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB115_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB115_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB115_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB115_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s1, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s1
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_sub_co_u32 v0, s0, v0, 1
; GFX12-NEXT:    s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v1, s0, 0, v1, s0
; GFX12-NEXT:    s_or_b32 s0, s0, vcc_lo
; GFX12-NEXT:    s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, s0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s1
; GFX12-NEXT:  .LBB115_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_dec_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_dec_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB116_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB116_3
; GCN1-NEXT:  .LBB116_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB116_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB116_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e64 v6, s[0:1], 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_subbrev_u32_e64 v7, s[0:1], 0, v1, s[0:1]
; GCN1-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB116_5: ; %atomicrmw.phi
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB116_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB116_3
; GCN2-NEXT:  .LBB116_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB116_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB116_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e64 v6, s[0:1], 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_subbrev_u32_e64 v7, s[0:1], 0, v1, s[0:1]
; GCN2-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB116_5: ; %atomicrmw.phi
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB116_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB116_3
; GFX12-NEXT:  .LBB116_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB116_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB116_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s1, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s1
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_sub_co_u32 v2, s0, v0, 1
; GFX12-NEXT:    s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v3, s0, 0, v1, s0
; GFX12-NEXT:    s_or_b32 s0, s0, vcc_lo
; GFX12-NEXT:    s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v3, s5, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v2, s4, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s1
; GFX12-NEXT:  .LBB116_5: ; %atomicrmw.phi
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr inbounds i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_dec_i64_decr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB117_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB117_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB117_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_subrev_i32_e64 v0, s[0:1], 1, v0
; GCN1-NEXT:    v_subbrev_u32_e64 v1, s[0:1], 0, v1, s[0:1]
; GCN1-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB117_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64_decr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xfc
; GCN2-NEXT:    s_add_u32 s0, s0, s6
; GCN2-NEXT:    s_addc_u32 s1, s1, s7
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB117_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB117_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB117_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_subrev_u32_e64 v0, s[0:1], 1, v0
; GCN2-NEXT:    v_subbrev_u32_e64 v1, s[0:1], 0, v1, s[0:1]
; GCN2-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB117_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_decr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB117_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB117_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB117_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s1, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s1
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_sub_co_u32 v0, s0, v0, 1
; GFX12-NEXT:    s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v1, s0, 0, v1, s0
; GFX12-NEXT:    s_or_b32 s0, s0, vcc_lo
; GFX12-NEXT:    s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, s0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s1
; GFX12-NEXT:  .LBB117_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_dec_i64_ret_decr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_dec_i64_ret_decr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB118_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB118_3
; GCN1-NEXT:  .LBB118_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB118_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB118_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e64 v6, s[0:1], 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_subbrev_u32_e64 v7, s[0:1], 0, v1, s[0:1]
; GCN1-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB118_5: ; %atomicrmw.phi
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64_ret_decr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB118_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB118_3
; GCN2-NEXT:  .LBB118_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB118_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB118_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e64 v6, s[0:1], 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_subbrev_u32_e64 v7, s[0:1], 0, v1, s[0:1]
; GCN2-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB118_5: ; %atomicrmw.phi
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_ret_decr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB118_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB118_3
; GFX12-NEXT:  .LBB118_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB118_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB118_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s1, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s1
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_sub_co_u32 v2, s0, v0, 1
; GFX12-NEXT:    s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v3, s0, 0, v1, s0
; GFX12-NEXT:    s_or_b32 s0, s0, vcc_lo
; GFX12-NEXT:    s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v3, s5, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v2, s4, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s1
; GFX12-NEXT:  .LBB118_5: ; %atomicrmw.phi
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %gep = getelementptr inbounds i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_dec_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_dec_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB119_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB119_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB119_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_subrev_i32_e64 v0, s[0:1], 1, v0
; GCN1-NEXT:    v_subbrev_u32_e64 v1, s[0:1], 0, v1, s[0:1]
; GCN1-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB119_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB119_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB119_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB119_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_subrev_u32_e64 v0, s[0:1], 1, v0
; GCN2-NEXT:    v_subbrev_u32_e64 v1, s[0:1], 0, v1, s[0:1]
; GCN2-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB119_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB119_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB119_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB119_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s1, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s1
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_sub_co_u32 v0, s0, v0, 1
; GFX12-NEXT:    s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v1, s0, 0, v1, s0
; GFX12-NEXT:    s_or_b32 s0, s0, vcc_lo
; GFX12-NEXT:    s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, s0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s1
; GFX12-NEXT:  .LBB119_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile udec_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_dec_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_dec_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB120_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:    s_branch .LBB120_3
; GCN1-NEXT:  .LBB120_2:
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB120_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s6, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s6, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB120_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e64 v6, s[0:1], 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_subbrev_u32_e64 v7, s[0:1], 0, v1, s[0:1]
; GCN1-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB120_5: ; %atomicrmw.phi
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB120_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:    s_branch .LBB120_3
; GCN2-NEXT:  .LBB120_2:
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB120_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN2-NEXT:    s_cselect_b32 s6, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s6, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB120_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e64 v6, s[0:1], 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_subbrev_u32_e64 v7, s[0:1], 0, v1, s[0:1]
; GCN2-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB120_5: ; %atomicrmw.phi
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB120_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB120_3
; GFX12-NEXT:  .LBB120_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB120_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB120_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s1, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s1
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_sub_co_u32 v2, s0, v0, 1
; GFX12-NEXT:    s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v3, s0, 0, v1, s0
; GFX12-NEXT:    s_or_b32 s0, s0, vcc_lo
; GFX12-NEXT:    s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v3, s5, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v2, s4, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s1
; GFX12-NEXT:  .LBB120_5: ; %atomicrmw.phi
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile udec_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_dec_i64_decr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_scc0 .LBB121_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB121_2: ; %Flow
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s4, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s4, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB121_4
; GCN1-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_subrev_i32_e64 v0, s[0:1], 1, v0
; GCN1-NEXT:    v_subbrev_u32_e64 v1, s[0:1], 0, v1, s[0:1]
; GCN1-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB121_4: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64_decr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_scc0 .LBB121_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB121_2: ; %Flow
; GCN2-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN2-NEXT:    s_cselect_b32 s4, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s4, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB121_4
; GCN2-NEXT:  ; %bb.3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_subrev_u32_e64 v0, s[0:1], 1, v0
; GCN2-NEXT:    v_subbrev_u32_e64 v1, s[0:1], 0, v1, s[0:1]
; GCN2-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB121_4: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_decr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[8:9], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[8:9], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_scc0 .LBB121_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_mov_b32 s4, 0
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:  .LBB121_2: ; %Flow
; GFX12-NEXT:    s_and_b32 s4, s4, exec_lo
; GFX12-NEXT:    s_cselect_b32 s4, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s4, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB121_4
; GFX12-NEXT:  ; %bb.3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s1, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s1
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_sub_co_u32 v0, s0, v0, 1
; GFX12-NEXT:    s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v1, s0, 0, v1, s0
; GFX12-NEXT:    s_or_b32 s0, s0, vcc_lo
; GFX12-NEXT:    s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, s0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s1
; GFX12-NEXT:  .LBB121_4: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile udec_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  ret void
}

define amdgpu_kernel void @atomic_dec_i64_ret_decr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_dec_i64_ret_decr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc0 .LBB122_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_mov_b64 s[2:3], 0
; GCN1-NEXT:    s_branch .LBB122_3
; GCN1-NEXT:  .LBB122_2:
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB122_3: ; %Flow
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, 1, 0
; GCN1-NEXT:    s_cmp_lg_u32 s2, 1
; GCN1-NEXT:    s_cbranch_scc1 .LBB122_5
; GCN1-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e64 v6, s[0:1], 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_subbrev_u32_e64 v7, s[0:1], 0, v1, s[0:1]
; GCN1-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB122_5: ; %atomicrmw.phi
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64_ret_decr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc0 .LBB122_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_mov_b64 s[2:3], 0
; GCN2-NEXT:    s_branch .LBB122_3
; GCN2-NEXT:  .LBB122_2:
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB122_3: ; %Flow
; GCN2-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN2-NEXT:    s_cselect_b32 s2, 1, 0
; GCN2-NEXT:    s_cmp_lg_u32 s2, 1
; GCN2-NEXT:    s_cbranch_scc1 .LBB122_5
; GCN2-NEXT:  ; %bb.4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e64 v6, s[0:1], 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_subbrev_u32_e64 v7, s[0:1], 0, v1, s[0:1]
; GCN2-NEXT:    s_or_b64 vcc, s[0:1], vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB122_5: ; %atomicrmw.phi
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_ret_decr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc0 .LBB122_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_mov_b32 s6, 0
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_branch .LBB122_3
; GFX12-NEXT:  .LBB122_2:
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB122_3: ; %Flow
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_b32 s6, s6, exec_lo
; GFX12-NEXT:    s_cselect_b32 s6, 1, 0
; GFX12-NEXT:    s_cmp_lg_u32 s6, 1
; GFX12-NEXT:    s_cbranch_scc1 .LBB122_5
; GFX12-NEXT:  ; %bb.4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s1, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s1
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_sub_co_u32 v2, s0, v0, 1
; GFX12-NEXT:    s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v3, s0, 0, v1, s0
; GFX12-NEXT:    s_or_b32 s0, s0, vcc_lo
; GFX12-NEXT:    s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v3, s5, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v2, s4, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s1
; GFX12-NEXT:  .LBB122_5: ; %atomicrmw.phi
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile udec_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
  store i64 %tmp0, ptr %out2
  ret void
}

!0 = !{}