llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefix=GCN1 %s
; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=GCN2 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s

define amdgpu_kernel void @atomic_add_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_add_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB0_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB0_4
; GCN1-NEXT:  .LBB0_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB0_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB0_2
; GCN1-NEXT:  .LBB0_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB0_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB0_4
; GCN2-NEXT:  .LBB0_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB0_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB0_2
; GCN2-NEXT:  .LBB0_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB0_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB0_4
; GFX12-NEXT:  .LBB0_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB0_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB0_2
; GFX12-NEXT:  .LBB0_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_add_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_add_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB1_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB1_3
; GCN1-NEXT:    s_branch .LBB1_4
; GCN1-NEXT:  .LBB1_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB1_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v5, vcc, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB1_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB1_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB1_3
; GCN2-NEXT:    s_branch .LBB1_4
; GCN2-NEXT:  .LBB1_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB1_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v5, vcc, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB1_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB1_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB1_3
; GFX12-NEXT:    s_branch .LBB1_4
; GFX12-NEXT:  .LBB1_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB1_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB1_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_add_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_add_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB2_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB2_4
; GCN1-NEXT:  .LBB2_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB2_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB2_2
; GCN1-NEXT:  .LBB2_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB2_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB2_4
; GCN2-NEXT:  .LBB2_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB2_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB2_2
; GCN2-NEXT:  .LBB2_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB2_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB2_4
; GFX12-NEXT:  .LBB2_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB2_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB2_2
; GFX12-NEXT:  .LBB2_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_add_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_add_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB3_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB3_3
; GCN1-NEXT:    s_branch .LBB3_4
; GCN1-NEXT:  .LBB3_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB3_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v5, vcc, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB3_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB3_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB3_3
; GCN2-NEXT:    s_branch .LBB3_4
; GCN2-NEXT:  .LBB3_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB3_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v5, vcc, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB3_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB3_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB3_3
; GFX12-NEXT:    s_branch .LBB3_4
; GFX12-NEXT:  .LBB3_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB3_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB3_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile add ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_add_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_add_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB4_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB4_4
; GCN1-NEXT:  .LBB4_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB4_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB4_2
; GCN1-NEXT:  .LBB4_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB4_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB4_4
; GCN2-NEXT:  .LBB4_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB4_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB4_2
; GCN2-NEXT:  .LBB4_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB4_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB4_4
; GFX12-NEXT:  .LBB4_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB4_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB4_2
; GFX12-NEXT:  .LBB4_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile add ptr %out, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_add_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_add_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB5_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB5_3
; GCN1-NEXT:    s_branch .LBB5_4
; GCN1-NEXT:  .LBB5_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB5_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v5, vcc, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB5_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB5_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB5_3
; GCN2-NEXT:    s_branch .LBB5_4
; GCN2-NEXT:  .LBB5_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB5_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v5, vcc, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB5_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB5_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB5_3
; GFX12-NEXT:    s_branch .LBB5_4
; GFX12-NEXT:  .LBB5_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB5_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB5_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile add ptr %out, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_add_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_add_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB6_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB6_4
; GCN1-NEXT:  .LBB6_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB6_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB6_2
; GCN1-NEXT:  .LBB6_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB6_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB6_4
; GCN2-NEXT:  .LBB6_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB6_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB6_2
; GCN2-NEXT:  .LBB6_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB6_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB6_4
; GFX12-NEXT:  .LBB6_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB6_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB6_2
; GFX12-NEXT:  .LBB6_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile add ptr %ptr, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_add_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_add_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB7_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB7_3
; GCN1-NEXT:    s_branch .LBB7_4
; GCN1-NEXT:  .LBB7_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB7_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v5, vcc, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB7_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_add_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB7_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB7_3
; GCN2-NEXT:    s_branch .LBB7_4
; GCN2-NEXT:  .LBB7_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB7_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v5, vcc, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB7_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_add_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB7_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_add_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB7_3
; GFX12-NEXT:    s_branch .LBB7_4
; GFX12-NEXT:  .LBB7_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB7_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB7_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile add ptr %ptr, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_and_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_and_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB8_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB8_4
; GCN1-NEXT:  .LBB8_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB8_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB8_2
; GCN1-NEXT:  .LBB8_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB8_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB8_4
; GCN2-NEXT:  .LBB8_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB8_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB8_2
; GCN2-NEXT:  .LBB8_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB8_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB8_4
; GFX12-NEXT:  .LBB8_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB8_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB8_2
; GFX12-NEXT:  .LBB8_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_and_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_and_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_and_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB9_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB9_3
; GCN1-NEXT:    s_branch .LBB9_4
; GCN1-NEXT:  .LBB9_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB9_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v4, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v5, s5, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB9_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB9_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB9_3
; GCN2-NEXT:    s_branch .LBB9_4
; GCN2-NEXT:  .LBB9_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB9_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v4, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v5, s5, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB9_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB9_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB9_3
; GFX12-NEXT:    s_branch .LBB9_4
; GFX12-NEXT:  .LBB9_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB9_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_and_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB9_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_and_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_and_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB10_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB10_4
; GCN1-NEXT:  .LBB10_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB10_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB10_2
; GCN1-NEXT:  .LBB10_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB10_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB10_4
; GCN2-NEXT:  .LBB10_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB10_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB10_2
; GCN2-NEXT:  .LBB10_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB10_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB10_4
; GFX12-NEXT:  .LBB10_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB10_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB10_2
; GFX12-NEXT:  .LBB10_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_and_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_and_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB11_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB11_3
; GCN1-NEXT:    s_branch .LBB11_4
; GCN1-NEXT:  .LBB11_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB11_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v4, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v5, s13, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB11_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB11_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB11_3
; GCN2-NEXT:    s_branch .LBB11_4
; GCN2-NEXT:  .LBB11_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB11_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v4, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v5, s13, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB11_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB11_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB11_3
; GFX12-NEXT:    s_branch .LBB11_4
; GFX12-NEXT:  .LBB11_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB11_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_and_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB11_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_and_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_and_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB12_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB12_4
; GCN1-NEXT:  .LBB12_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB12_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB12_2
; GCN1-NEXT:  .LBB12_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB12_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB12_4
; GCN2-NEXT:  .LBB12_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB12_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB12_2
; GCN2-NEXT:  .LBB12_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB12_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB12_4
; GFX12-NEXT:  .LBB12_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB12_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB12_2
; GFX12-NEXT:  .LBB12_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_and_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile and ptr %out, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_and_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_and_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB13_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB13_3
; GCN1-NEXT:    s_branch .LBB13_4
; GCN1-NEXT:  .LBB13_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB13_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v4, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v5, s5, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB13_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB13_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB13_3
; GCN2-NEXT:    s_branch .LBB13_4
; GCN2-NEXT:  .LBB13_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB13_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v4, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v5, s5, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB13_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB13_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB13_3
; GFX12-NEXT:    s_branch .LBB13_4
; GFX12-NEXT:  .LBB13_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB13_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_and_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB13_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile and ptr %out, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_and_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_and_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB14_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB14_4
; GCN1-NEXT:  .LBB14_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB14_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB14_2
; GCN1-NEXT:  .LBB14_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB14_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB14_4
; GCN2-NEXT:  .LBB14_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB14_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB14_2
; GCN2-NEXT:  .LBB14_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB14_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB14_4
; GFX12-NEXT:  .LBB14_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB14_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB14_2
; GFX12-NEXT:  .LBB14_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_and_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile and ptr %ptr, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_and_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_and_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB15_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB15_3
; GCN1-NEXT:    s_branch .LBB15_4
; GCN1-NEXT:  .LBB15_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB15_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_and_b32_e32 v4, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v5, s13, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB15_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_and_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB15_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB15_3
; GCN2-NEXT:    s_branch .LBB15_4
; GCN2-NEXT:  .LBB15_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB15_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_and_b32_e32 v4, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v5, s13, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB15_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_and_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB15_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_and_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB15_3
; GFX12-NEXT:    s_branch .LBB15_4
; GFX12-NEXT:  .LBB15_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB15_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_and_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_and_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB15_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile and ptr %ptr, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_sub_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_sub_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB16_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB16_4
; GCN1-NEXT:  .LBB16_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB16_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB16_2
; GCN1-NEXT:  .LBB16_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB16_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB16_4
; GCN2-NEXT:  .LBB16_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB16_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB16_2
; GCN2-NEXT:  .LBB16_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB16_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB16_4
; GFX12-NEXT:  .LBB16_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB16_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB16_2
; GFX12-NEXT:  .LBB16_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    v_subrev_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_sub_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_sub_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB17_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB17_3
; GCN1-NEXT:    s_branch .LBB17_4
; GCN1-NEXT:  .LBB17_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB17_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB17_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB17_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB17_3
; GCN2-NEXT:    s_branch .LBB17_4
; GCN2-NEXT:  .LBB17_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB17_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v5, vcc, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB17_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB17_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB17_3
; GFX12-NEXT:    s_branch .LBB17_4
; GFX12-NEXT:  .LBB17_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB17_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    v_subrev_co_ci_u32_e32 v3, vcc_lo, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB17_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_sub_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_sub_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB18_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB18_4
; GCN1-NEXT:  .LBB18_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB18_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB18_2
; GCN1-NEXT:  .LBB18_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB18_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB18_4
; GCN2-NEXT:  .LBB18_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB18_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB18_2
; GCN2-NEXT:  .LBB18_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB18_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB18_4
; GFX12-NEXT:  .LBB18_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB18_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB18_2
; GFX12-NEXT:  .LBB18_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    v_subrev_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_sub_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB19_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB19_3
; GCN1-NEXT:    s_branch .LBB19_4
; GCN1-NEXT:  .LBB19_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB19_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v5, vcc, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB19_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB19_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB19_3
; GCN2-NEXT:    s_branch .LBB19_4
; GCN2-NEXT:  .LBB19_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB19_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v5, vcc, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB19_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB19_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB19_3
; GFX12-NEXT:    s_branch .LBB19_4
; GFX12-NEXT:  .LBB19_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB19_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    v_subrev_co_ci_u32_e32 v3, vcc_lo, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB19_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_sub_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_sub_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB20_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB20_4
; GCN1-NEXT:  .LBB20_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB20_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB20_2
; GCN1-NEXT:  .LBB20_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB20_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB20_4
; GCN2-NEXT:  .LBB20_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB20_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB20_2
; GCN2-NEXT:  .LBB20_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB20_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB20_4
; GFX12-NEXT:  .LBB20_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB20_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB20_2
; GFX12-NEXT:  .LBB20_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    v_subrev_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile sub ptr %out, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_sub_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_sub_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB21_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB21_3
; GCN1-NEXT:    s_branch .LBB21_4
; GCN1-NEXT:  .LBB21_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB21_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB21_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB21_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB21_3
; GCN2-NEXT:    s_branch .LBB21_4
; GCN2-NEXT:  .LBB21_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB21_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v5, vcc, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB21_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB21_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB21_3
; GFX12-NEXT:    s_branch .LBB21_4
; GFX12-NEXT:  .LBB21_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB21_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    v_subrev_co_ci_u32_e32 v3, vcc_lo, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB21_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile sub ptr %out, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_sub_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_sub_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB22_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB22_4
; GCN1-NEXT:  .LBB22_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB22_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB22_2
; GCN1-NEXT:  .LBB22_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    buffer_load_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v1, vcc, s2, v1
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT:    buffer_store_dword v1, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v2, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB22_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB22_4
; GCN2-NEXT:  .LBB22_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB22_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB22_2
; GCN2-NEXT:  .LBB22_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    buffer_load_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v1, vcc, s2, v1
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT:    buffer_store_dword v1, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v2, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB22_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB22_4
; GFX12-NEXT:  .LBB22_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB22_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB22_2
; GFX12-NEXT:  .LBB22_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, s2
; GFX12-NEXT:    v_subrev_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile sub ptr %ptr, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_sub_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_sub_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB23_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB23_3
; GCN1-NEXT:    s_branch .LBB23_4
; GCN1-NEXT:  .LBB23_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB23_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_subrev_i32_e32 v5, vcc, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB23_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_sub_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB23_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB23_3
; GCN2-NEXT:    s_branch .LBB23_4
; GCN2-NEXT:  .LBB23_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB23_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_subrev_u32_e32 v5, vcc, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB23_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB23_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_sub_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB23_3
; GFX12-NEXT:    s_branch .LBB23_4
; GFX12-NEXT:  .LBB23_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB23_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_co_u32 v2, vcc_lo, v0, s4
; GFX12-NEXT:    v_subrev_co_ci_u32_e32 v3, vcc_lo, s5, v1, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB23_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile sub ptr %ptr, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_max_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_max_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB24_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB24_4
; GCN1-NEXT:  .LBB24_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB24_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB24_2
; GCN1-NEXT:  .LBB24_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB24_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB24_4
; GCN2-NEXT:  .LBB24_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB24_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB24_2
; GCN2-NEXT:  .LBB24_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB24_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB24_4
; GFX12-NEXT:  .LBB24_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB24_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB24_2
; GFX12-NEXT:  .LBB24_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_max_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_max_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB25_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB25_3
; GCN1-NEXT:    s_branch .LBB25_4
; GCN1-NEXT:  .LBB25_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB25_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB25_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB25_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB25_3
; GCN2-NEXT:    s_branch .LBB25_4
; GCN2-NEXT:  .LBB25_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB25_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB25_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB25_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB25_3
; GFX12-NEXT:    s_branch .LBB25_4
; GFX12-NEXT:  .LBB25_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB25_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB25_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_max_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB26_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB26_4
; GCN1-NEXT:  .LBB26_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB26_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB26_2
; GCN1-NEXT:  .LBB26_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB26_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB26_4
; GCN2-NEXT:  .LBB26_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB26_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB26_2
; GCN2-NEXT:  .LBB26_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB26_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB26_4
; GFX12-NEXT:  .LBB26_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB26_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB26_2
; GFX12-NEXT:  .LBB26_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_max_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB27_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB27_3
; GCN1-NEXT:    s_branch .LBB27_4
; GCN1-NEXT:  .LBB27_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB27_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB27_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB27_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB27_3
; GCN2-NEXT:    s_branch .LBB27_4
; GCN2-NEXT:  .LBB27_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB27_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB27_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB27_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB27_3
; GFX12-NEXT:    s_branch .LBB27_4
; GFX12-NEXT:  .LBB27_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB27_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB27_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_max_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_max_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB28_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB28_4
; GCN1-NEXT:  .LBB28_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB28_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB28_2
; GCN1-NEXT:  .LBB28_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB28_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB28_4
; GCN2-NEXT:  .LBB28_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB28_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB28_2
; GCN2-NEXT:  .LBB28_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB28_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB28_4
; GFX12-NEXT:  .LBB28_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB28_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB28_2
; GFX12-NEXT:  .LBB28_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile max ptr %out, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_max_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_max_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB29_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB29_3
; GCN1-NEXT:    s_branch .LBB29_4
; GCN1-NEXT:  .LBB29_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB29_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB29_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB29_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB29_3
; GCN2-NEXT:    s_branch .LBB29_4
; GCN2-NEXT:  .LBB29_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB29_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB29_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB29_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB29_3
; GFX12-NEXT:    s_branch .LBB29_4
; GFX12-NEXT:  .LBB29_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB29_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB29_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile max ptr %out, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_max_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB30_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB30_4
; GCN1-NEXT:  .LBB30_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB30_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB30_2
; GCN1-NEXT:  .LBB30_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB30_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB30_4
; GCN2-NEXT:  .LBB30_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB30_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB30_2
; GCN2-NEXT:  .LBB30_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB30_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB30_4
; GFX12-NEXT:  .LBB30_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB30_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB30_2
; GFX12-NEXT:  .LBB30_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile max ptr %ptr, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_max_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB31_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB31_3
; GCN1-NEXT:    s_branch .LBB31_4
; GCN1-NEXT:  .LBB31_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB31_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB31_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB31_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB31_3
; GCN2-NEXT:    s_branch .LBB31_4
; GCN2-NEXT:  .LBB31_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB31_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB31_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_max_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB31_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_max_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB31_3
; GFX12-NEXT:    s_branch .LBB31_4
; GFX12-NEXT:  .LBB31_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB31_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB31_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile max ptr %ptr, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umax_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_umax_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB32_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB32_4
; GCN1-NEXT:  .LBB32_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB32_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB32_2
; GCN1-NEXT:  .LBB32_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB32_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB32_4
; GCN2-NEXT:  .LBB32_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB32_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB32_2
; GCN2-NEXT:  .LBB32_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB32_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB32_4
; GFX12-NEXT:  .LBB32_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB32_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB32_2
; GFX12-NEXT:  .LBB32_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_umax_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_umax_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB33_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB33_3
; GCN1-NEXT:    s_branch .LBB33_4
; GCN1-NEXT:  .LBB33_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB33_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB33_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB33_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB33_3
; GCN2-NEXT:    s_branch .LBB33_4
; GCN2-NEXT:  .LBB33_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB33_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB33_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB33_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB33_3
; GFX12-NEXT:    s_branch .LBB33_4
; GFX12-NEXT:  .LBB33_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB33_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB33_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umax_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB34_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB34_4
; GCN1-NEXT:  .LBB34_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB34_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB34_2
; GCN1-NEXT:  .LBB34_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB34_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB34_4
; GCN2-NEXT:  .LBB34_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB34_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB34_2
; GCN2-NEXT:  .LBB34_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB34_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB34_4
; GFX12-NEXT:  .LBB34_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB34_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB34_2
; GFX12-NEXT:  .LBB34_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umax_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB35_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB35_3
; GCN1-NEXT:    s_branch .LBB35_4
; GCN1-NEXT:  .LBB35_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB35_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB35_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB35_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB35_3
; GCN2-NEXT:    s_branch .LBB35_4
; GCN2-NEXT:  .LBB35_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB35_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB35_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB35_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB35_3
; GFX12-NEXT:    s_branch .LBB35_4
; GFX12-NEXT:  .LBB35_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB35_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB35_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umax_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_umax_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB36_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB36_4
; GCN1-NEXT:  .LBB36_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB36_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB36_2
; GCN1-NEXT:  .LBB36_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB36_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB36_4
; GCN2-NEXT:  .LBB36_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB36_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB36_2
; GCN2-NEXT:  .LBB36_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB36_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB36_4
; GFX12-NEXT:  .LBB36_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB36_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB36_2
; GFX12-NEXT:  .LBB36_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile umax ptr %out, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_umax_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_umax_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB37_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB37_3
; GCN1-NEXT:    s_branch .LBB37_4
; GCN1-NEXT:  .LBB37_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB37_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB37_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB37_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB37_3
; GCN2-NEXT:    s_branch .LBB37_4
; GCN2-NEXT:  .LBB37_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB37_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB37_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB37_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB37_3
; GFX12-NEXT:    s_branch .LBB37_4
; GFX12-NEXT:  .LBB37_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB37_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB37_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile umax ptr %out, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umax_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umax_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB38_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB38_4
; GCN1-NEXT:  .LBB38_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB38_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB38_2
; GCN1-NEXT:  .LBB38_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB38_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB38_4
; GCN2-NEXT:  .LBB38_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB38_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB38_2
; GCN2-NEXT:  .LBB38_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB38_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB38_4
; GFX12-NEXT:  .LBB38_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB38_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB38_2
; GFX12-NEXT:  .LBB38_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile umax ptr %ptr, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umax_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB39_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB39_3
; GCN1-NEXT:    s_branch .LBB39_4
; GCN1-NEXT:  .LBB39_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB39_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB39_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB39_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB39_3
; GCN2-NEXT:    s_branch .LBB39_4
; GCN2-NEXT:  .LBB39_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB39_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB39_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB39_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_max_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB39_3
; GFX12-NEXT:    s_branch .LBB39_4
; GFX12-NEXT:  .LBB39_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB39_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB39_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile umax ptr %ptr, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_min_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_min_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB40_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB40_4
; GCN1-NEXT:  .LBB40_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB40_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB40_2
; GCN1-NEXT:  .LBB40_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB40_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB40_4
; GCN2-NEXT:  .LBB40_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB40_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB40_2
; GCN2-NEXT:  .LBB40_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB40_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB40_4
; GFX12-NEXT:  .LBB40_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB40_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB40_2
; GFX12-NEXT:  .LBB40_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_min_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_min_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB41_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB41_3
; GCN1-NEXT:    s_branch .LBB41_4
; GCN1-NEXT:  .LBB41_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB41_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB41_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB41_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB41_3
; GCN2-NEXT:    s_branch .LBB41_4
; GCN2-NEXT:  .LBB41_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB41_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB41_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB41_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB41_3
; GFX12-NEXT:    s_branch .LBB41_4
; GFX12-NEXT:  .LBB41_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB41_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB41_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_min_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB42_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB42_4
; GCN1-NEXT:  .LBB42_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB42_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB42_2
; GCN1-NEXT:  .LBB42_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB42_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB42_4
; GCN2-NEXT:  .LBB42_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB42_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB42_2
; GCN2-NEXT:  .LBB42_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB42_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB42_4
; GFX12-NEXT:  .LBB42_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB42_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB42_2
; GFX12-NEXT:  .LBB42_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_min_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB43_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB43_3
; GCN1-NEXT:    s_branch .LBB43_4
; GCN1-NEXT:  .LBB43_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB43_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB43_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB43_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB43_3
; GCN2-NEXT:    s_branch .LBB43_4
; GCN2-NEXT:  .LBB43_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB43_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB43_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB43_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB43_3
; GFX12-NEXT:    s_branch .LBB43_4
; GFX12-NEXT:  .LBB43_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB43_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB43_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_min_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB44_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB44_4
; GCN1-NEXT:  .LBB44_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB44_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB44_2
; GCN1-NEXT:  .LBB44_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB44_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB44_4
; GCN2-NEXT:  .LBB44_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB44_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB44_2
; GCN2-NEXT:  .LBB44_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB44_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB44_4
; GFX12-NEXT:  .LBB44_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB44_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB44_2
; GFX12-NEXT:  .LBB44_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile min ptr %out, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_min_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_min_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB45_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB45_3
; GCN1-NEXT:    s_branch .LBB45_4
; GCN1-NEXT:  .LBB45_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB45_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB45_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB45_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB45_3
; GCN2-NEXT:    s_branch .LBB45_4
; GCN2-NEXT:  .LBB45_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB45_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB45_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB45_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB45_3
; GFX12-NEXT:    s_branch .LBB45_4
; GFX12-NEXT:  .LBB45_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB45_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB45_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile min ptr %out, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_min_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_min_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB46_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB46_4
; GCN1-NEXT:  .LBB46_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB46_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB46_2
; GCN1-NEXT:  .LBB46_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB46_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB46_4
; GCN2-NEXT:  .LBB46_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB46_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB46_2
; GCN2-NEXT:  .LBB46_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB46_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB46_4
; GFX12-NEXT:  .LBB46_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB46_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB46_2
; GFX12-NEXT:  .LBB46_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile min ptr %ptr, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_min_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB47_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB47_3
; GCN1-NEXT:    s_branch .LBB47_4
; GCN1-NEXT:  .LBB47_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB47_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB47_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB47_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB47_3
; GCN2-NEXT:    s_branch .LBB47_4
; GCN2-NEXT:  .LBB47_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB47_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB47_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_min_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB47_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_min_i64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB47_3
; GFX12-NEXT:    s_branch .LBB47_4
; GFX12-NEXT:  .LBB47_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB47_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_i64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB47_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile min ptr %ptr, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umin_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_umin_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB48_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB48_4
; GCN1-NEXT:  .LBB48_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB48_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB48_2
; GCN1-NEXT:  .LBB48_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB48_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB48_4
; GCN2-NEXT:  .LBB48_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB48_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB48_2
; GCN2-NEXT:  .LBB48_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB48_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB48_4
; GFX12-NEXT:  .LBB48_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB48_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB48_2
; GFX12-NEXT:  .LBB48_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_umin_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_umin_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB49_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB49_3
; GCN1-NEXT:    s_branch .LBB49_4
; GCN1-NEXT:  .LBB49_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB49_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB49_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB49_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB49_3
; GCN2-NEXT:    s_branch .LBB49_4
; GCN2-NEXT:  .LBB49_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB49_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB49_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB49_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB49_3
; GFX12-NEXT:    s_branch .LBB49_4
; GFX12-NEXT:  .LBB49_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB49_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB49_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umin_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umin_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB50_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB50_4
; GCN1-NEXT:  .LBB50_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB50_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB50_2
; GCN1-NEXT:  .LBB50_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB50_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB50_4
; GCN2-NEXT:  .LBB50_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB50_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB50_2
; GCN2-NEXT:  .LBB50_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB50_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB50_4
; GFX12-NEXT:  .LBB50_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB50_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB50_2
; GFX12-NEXT:  .LBB50_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umin_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB51_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB51_3
; GCN1-NEXT:    s_branch .LBB51_4
; GCN1-NEXT:  .LBB51_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB51_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB51_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB51_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB51_3
; GCN2-NEXT:    s_branch .LBB51_4
; GCN2-NEXT:  .LBB51_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB51_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB51_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB51_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB51_3
; GFX12-NEXT:    s_branch .LBB51_4
; GFX12-NEXT:  .LBB51_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB51_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB51_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umin_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_umin_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB52_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB52_4
; GCN1-NEXT:  .LBB52_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB52_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB52_2
; GCN1-NEXT:  .LBB52_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB52_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB52_4
; GCN2-NEXT:  .LBB52_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB52_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB52_2
; GCN2-NEXT:  .LBB52_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB52_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB52_4
; GFX12-NEXT:  .LBB52_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB52_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB52_2
; GFX12-NEXT:  .LBB52_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile umin ptr %out, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_umin_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_umin_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB53_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB53_3
; GCN1-NEXT:    s_branch .LBB53_4
; GCN1-NEXT:  .LBB53_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB53_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s5
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB53_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB53_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB53_3
; GCN2-NEXT:    s_branch .LBB53_4
; GCN2-NEXT:  .LBB53_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB53_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s4
; GCN2-NEXT:    v_mov_b32_e32 v4, s5
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB53_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB53_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB53_3
; GFX12-NEXT:    s_branch .LBB53_4
; GFX12-NEXT:  .LBB53_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB53_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB53_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile umin ptr %out, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umin_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umin_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB54_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB54_4
; GCN1-NEXT:  .LBB54_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB54_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execnz .LBB54_2
; GCN1-NEXT:  .LBB54_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB54_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB54_4
; GCN2-NEXT:  .LBB54_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB54_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execnz .LBB54_2
; GCN2-NEXT:  .LBB54_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB54_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB54_4
; GFX12-NEXT:  .LBB54_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB54_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[2:3] scope:SCOPE_SE
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execnz .LBB54_2
; GFX12-NEXT:  .LBB54_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v1, s3, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v0, s2, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile umin ptr %ptr, i64 %in syncscope("workgroup") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_umin_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umin_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB55_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cbranch_execz .LBB55_3
; GCN1-NEXT:    s_branch .LBB55_4
; GCN1-NEXT:  .LBB55_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB55_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_ge_u64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB55_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umin_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB55_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cbranch_execz .LBB55_3
; GCN2-NEXT:    s_branch .LBB55_4
; GCN2-NEXT:  .LBB55_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB55_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_ge_u64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB55_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB55_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_min_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SE
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SE
; GFX12-NEXT:    s_cbranch_execz .LBB55_3
; GFX12-NEXT:    s_branch .LBB55_4
; GFX12-NEXT:  .LBB55_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB55_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_ge_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e32 v3, s5, v1, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e32 v2, s4, v0, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB55_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile umin ptr %ptr, i64 %in syncscope("workgroup") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_or_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_or_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB56_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB56_4
; GCN1-NEXT:  .LBB56_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB56_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB56_2
; GCN1-NEXT:  .LBB56_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB56_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB56_4
; GCN2-NEXT:  .LBB56_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB56_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB56_2
; GCN2-NEXT:  .LBB56_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB56_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB56_4
; GFX12-NEXT:  .LBB56_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB56_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB56_2
; GFX12-NEXT:  .LBB56_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_or_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_or_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_or_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB57_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB57_3
; GCN1-NEXT:    s_branch .LBB57_4
; GCN1-NEXT:  .LBB57_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB57_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v4, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v5, s5, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB57_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB57_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB57_3
; GCN2-NEXT:    s_branch .LBB57_4
; GCN2-NEXT:  .LBB57_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB57_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v4, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v5, s5, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB57_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB57_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB57_3
; GFX12-NEXT:    s_branch .LBB57_4
; GFX12-NEXT:  .LBB57_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB57_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_or_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB57_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_or_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_or_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB58_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB58_4
; GCN1-NEXT:  .LBB58_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB58_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB58_2
; GCN1-NEXT:  .LBB58_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB58_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB58_4
; GCN2-NEXT:  .LBB58_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB58_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB58_2
; GCN2-NEXT:  .LBB58_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB58_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB58_4
; GFX12-NEXT:  .LBB58_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB58_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB58_2
; GFX12-NEXT:  .LBB58_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_or_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_or_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB59_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB59_3
; GCN1-NEXT:    s_branch .LBB59_4
; GCN1-NEXT:  .LBB59_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB59_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v4, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v5, s13, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB59_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB59_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB59_3
; GCN2-NEXT:    s_branch .LBB59_4
; GCN2-NEXT:  .LBB59_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB59_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v4, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v5, s13, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB59_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB59_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB59_3
; GFX12-NEXT:    s_branch .LBB59_4
; GFX12-NEXT:  .LBB59_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB59_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_or_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB59_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_or_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_or_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB60_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB60_4
; GCN1-NEXT:  .LBB60_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB60_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB60_2
; GCN1-NEXT:  .LBB60_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB60_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB60_4
; GCN2-NEXT:  .LBB60_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB60_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB60_2
; GCN2-NEXT:  .LBB60_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB60_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB60_4
; GFX12-NEXT:  .LBB60_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB60_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB60_2
; GFX12-NEXT:  .LBB60_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_or_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile or ptr %out, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_or_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_or_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB61_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB61_3
; GCN1-NEXT:    s_branch .LBB61_4
; GCN1-NEXT:  .LBB61_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB61_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v4, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v5, s5, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB61_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB61_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB61_3
; GCN2-NEXT:    s_branch .LBB61_4
; GCN2-NEXT:  .LBB61_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB61_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v4, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v5, s5, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB61_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB61_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB61_3
; GFX12-NEXT:    s_branch .LBB61_4
; GFX12-NEXT:  .LBB61_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB61_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_or_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB61_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile or ptr %out, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_or_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_or_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB62_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB62_4
; GCN1-NEXT:  .LBB62_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB62_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB62_2
; GCN1-NEXT:  .LBB62_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB62_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB62_4
; GCN2-NEXT:  .LBB62_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB62_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB62_2
; GCN2-NEXT:  .LBB62_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB62_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB62_4
; GFX12-NEXT:  .LBB62_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB62_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB62_2
; GFX12-NEXT:  .LBB62_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_or_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile or ptr %ptr, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_or_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_or_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB63_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB63_3
; GCN1-NEXT:    s_branch .LBB63_4
; GCN1-NEXT:  .LBB63_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB63_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_or_b32_e32 v4, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v5, s13, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB63_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_or_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB63_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB63_3
; GCN2-NEXT:    s_branch .LBB63_4
; GCN2-NEXT:  .LBB63_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB63_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_or_b32_e32 v4, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v5, s13, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB63_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_or_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB63_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_or_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB63_3
; GFX12-NEXT:    s_branch .LBB63_4
; GFX12-NEXT:  .LBB63_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB63_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_or_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_or_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB63_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile or ptr %ptr, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_xchg_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB64_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB64_4
; GCN1-NEXT:  .LBB64_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB64_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB64_2
; GCN1-NEXT:  .LBB64_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v0, s3
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB64_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB64_4
; GCN2-NEXT:  .LBB64_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB64_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB64_2
; GCN2-NEXT:  .LBB64_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v0, s3
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB64_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB64_4
; GFX12-NEXT:  .LBB64_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB64_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB64_2
; GFX12-NEXT:  .LBB64_4: ; %atomicrmw.private
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xchg_f64_offset(ptr %out, double %in) {
; GCN1-LABEL: atomic_xchg_f64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB65_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB65_4
; GCN1-NEXT:  .LBB65_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB65_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB65_2
; GCN1-NEXT:  .LBB65_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v0, s3
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_f64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB65_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB65_4
; GCN2-NEXT:  .LBB65_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB65_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB65_2
; GCN2-NEXT:  .LBB65_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v0, s3
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_f64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB65_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB65_4
; GFX12-NEXT:  .LBB65_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB65_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB65_2
; GFX12-NEXT:  .LBB65_4: ; %atomicrmw.private
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr double, ptr %out, i64 4
  %tmp0 = atomicrmw volatile xchg ptr %gep, double %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xchg_pointer_offset(ptr %out, ptr %in) {
; GCN1-LABEL: atomic_xchg_pointer_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB66_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB66_4
; GCN1-NEXT:  .LBB66_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB66_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB66_2
; GCN1-NEXT:  .LBB66_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v0, s3
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_pointer_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB66_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB66_4
; GCN2-NEXT:  .LBB66_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB66_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB66_2
; GCN2-NEXT:  .LBB66_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v0, s3
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_pointer_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB66_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB66_4
; GFX12-NEXT:  .LBB66_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB66_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB66_2
; GFX12-NEXT:  .LBB66_4: ; %atomicrmw.private
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr ptr, ptr %out, i32 4
  %val = atomicrmw volatile xchg ptr %gep, ptr %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_xchg_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB67_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB67_3
; GCN1-NEXT:    s_branch .LBB67_4
; GCN1-NEXT:  .LBB67_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB67_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:    buffer_store_dword v2, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB67_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(2)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB67_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB67_3
; GCN2-NEXT:    s_branch .LBB67_4
; GCN2-NEXT:  .LBB67_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB67_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s4
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:    buffer_store_dword v2, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB67_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(2)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB67_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB67_3
; GFX12-NEXT:    s_branch .LBB67_4
; GFX12-NEXT:  .LBB67_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB67_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB67_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xchg_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB68_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB68_4
; GCN1-NEXT:  .LBB68_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB68_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB68_2
; GCN1-NEXT:  .LBB68_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v0, s3
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB68_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB68_4
; GCN2-NEXT:  .LBB68_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB68_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB68_2
; GCN2-NEXT:  .LBB68_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v0, s3
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB68_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB68_4
; GFX12-NEXT:  .LBB68_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB68_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB68_2
; GFX12-NEXT:  .LBB68_4: ; %atomicrmw.private
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xchg_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB69_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB69_3
; GCN1-NEXT:    s_branch .LBB69_4
; GCN1-NEXT:  .LBB69_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB69_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s13
; GCN1-NEXT:    buffer_store_dword v2, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB69_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(2)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB69_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB69_3
; GCN2-NEXT:    s_branch .LBB69_4
; GCN2-NEXT:  .LBB69_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB69_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s12
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s13
; GCN2-NEXT:    buffer_store_dword v2, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB69_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(2)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB69_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB69_3
; GFX12-NEXT:    s_branch .LBB69_4
; GFX12-NEXT:  .LBB69_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB69_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB69_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile xchg ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_xchg_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB70_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB70_4
; GCN1-NEXT:  .LBB70_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB70_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB70_2
; GCN1-NEXT:  .LBB70_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v0, s3
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB70_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB70_4
; GCN2-NEXT:  .LBB70_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB70_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB70_2
; GCN2-NEXT:  .LBB70_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v0, s3
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB70_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB70_4
; GFX12-NEXT:  .LBB70_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB70_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB70_2
; GFX12-NEXT:  .LBB70_4: ; %atomicrmw.private
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile xchg ptr %out, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_xchg_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB71_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB71_3
; GCN1-NEXT:    s_branch .LBB71_4
; GCN1-NEXT:  .LBB71_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB71_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s4
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:    buffer_store_dword v2, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB71_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    s_waitcnt vmcnt(2)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB71_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB71_3
; GCN2-NEXT:    s_branch .LBB71_4
; GCN2-NEXT:  .LBB71_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB71_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s4
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:    buffer_store_dword v2, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB71_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    s_waitcnt vmcnt(2)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB71_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB71_3
; GFX12-NEXT:    s_branch .LBB71_4
; GFX12-NEXT:  .LBB71_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB71_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB71_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile xchg ptr %out, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xchg_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB72_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB72_4
; GCN1-NEXT:  .LBB72_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB72_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB72_2
; GCN1-NEXT:  .LBB72_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v0, s3
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB72_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB72_4
; GCN2-NEXT:  .LBB72_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB72_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB72_2
; GCN2-NEXT:  .LBB72_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v0, s3
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB72_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB72_4
; GFX12-NEXT:  .LBB72_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB72_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB72_2
; GFX12-NEXT:  .LBB72_4: ; %atomicrmw.private
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile xchg ptr %ptr, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xchg_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xchg_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB73_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB73_3
; GCN1-NEXT:    s_branch .LBB73_4
; GCN1-NEXT:  .LBB73_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB73_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v2, s13
; GCN1-NEXT:    buffer_store_dword v2, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB73_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    s_waitcnt vmcnt(2)
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xchg_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB73_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB73_3
; GCN2-NEXT:    s_branch .LBB73_4
; GCN2-NEXT:  .LBB73_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB73_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s12
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v2, s13
; GCN2-NEXT:    buffer_store_dword v2, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB73_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    s_waitcnt vmcnt(2)
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xchg_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB73_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_swap_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB73_3
; GFX12-NEXT:    s_branch .LBB73_4
; GFX12-NEXT:  .LBB73_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB73_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB73_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile xchg ptr %ptr, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xor_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_xor_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB74_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB74_4
; GCN1-NEXT:  .LBB74_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB74_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB74_2
; GCN1-NEXT:  .LBB74_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB74_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB74_4
; GCN2-NEXT:  .LBB74_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB74_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB74_2
; GCN2-NEXT:  .LBB74_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB74_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB74_4
; GFX12-NEXT:  .LBB74_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB74_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB74_2
; GFX12-NEXT:  .LBB74_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_xor_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xor_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_xor_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB75_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB75_3
; GCN1-NEXT:    s_branch .LBB75_4
; GCN1-NEXT:  .LBB75_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB75_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v4, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v5, s5, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB75_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB75_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB75_3
; GCN2-NEXT:    s_branch .LBB75_4
; GCN2-NEXT:  .LBB75_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB75_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v4, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v5, s5, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB75_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB75_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB75_3
; GFX12-NEXT:    s_branch .LBB75_4
; GFX12-NEXT:  .LBB75_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB75_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_xor_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB75_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xor_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xor_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB76_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB76_4
; GCN1-NEXT:  .LBB76_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB76_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB76_2
; GCN1-NEXT:  .LBB76_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB76_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB76_4
; GCN2-NEXT:  .LBB76_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB76_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB76_2
; GCN2-NEXT:  .LBB76_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB76_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB76_4
; GFX12-NEXT:  .LBB76_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB76_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB76_2
; GFX12-NEXT:  .LBB76_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_xor_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xor_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB77_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB77_3
; GCN1-NEXT:    s_branch .LBB77_4
; GCN1-NEXT:  .LBB77_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB77_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v4, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v5, s13, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB77_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB77_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB77_3
; GCN2-NEXT:    s_branch .LBB77_4
; GCN2-NEXT:  .LBB77_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB77_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v4, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v5, s13, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB77_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB77_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB77_3
; GFX12-NEXT:    s_branch .LBB77_4
; GFX12-NEXT:  .LBB77_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB77_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_xor_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB77_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xor_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_xor_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB78_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB78_4
; GCN1-NEXT:  .LBB78_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB78_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB78_2
; GCN1-NEXT:  .LBB78_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB78_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB78_4
; GCN2-NEXT:  .LBB78_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB78_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB78_2
; GCN2-NEXT:  .LBB78_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB78_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB78_4
; GFX12-NEXT:  .LBB78_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB78_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB78_2
; GFX12-NEXT:  .LBB78_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_xor_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile xor ptr %out, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xor_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_xor_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB79_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB79_3
; GCN1-NEXT:    s_branch .LBB79_4
; GCN1-NEXT:  .LBB79_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB79_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v4, s4, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v5, s5, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB79_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB79_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB79_3
; GCN2-NEXT:    s_branch .LBB79_4
; GCN2-NEXT:  .LBB79_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB79_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v4, s4, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v5, s5, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB79_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB79_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB79_3
; GFX12-NEXT:    s_branch .LBB79_4
; GFX12-NEXT:  .LBB79_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB79_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_xor_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB79_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile xor ptr %out, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_xor_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xor_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB80_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB80_4
; GCN1-NEXT:  .LBB80_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB80_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB80_2
; GCN1-NEXT:  .LBB80_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v1, s0
; GCN1-NEXT:    buffer_load_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN1-NEXT:    buffer_store_dword v2, v0, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v3, v1, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB80_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB80_4
; GCN2-NEXT:  .LBB80_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB80_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB80_2
; GCN2-NEXT:  .LBB80_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v1, s0
; GCN2-NEXT:    buffer_load_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v2, s2, v2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v3, s3, v3
; GCN2-NEXT:    buffer_store_dword v2, v0, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v3, v1, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB80_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB80_4
; GFX12-NEXT:  .LBB80_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB80_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB80_2
; GFX12-NEXT:  .LBB80_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v1, s3, v1
; GFX12-NEXT:    v_xor_b32_e32 v0, s2, v0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile xor ptr %ptr, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_xor_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_xor_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB81_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB81_3
; GCN1-NEXT:    s_branch .LBB81_4
; GCN1-NEXT:  .LBB81_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB81_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_xor_b32_e32 v4, s12, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v5, s13, v1
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB81_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_xor_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB81_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB81_3
; GCN2-NEXT:    s_branch .LBB81_4
; GCN2-NEXT:  .LBB81_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB81_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_xor_b32_e32 v4, s12, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v5, s13, v1
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB81_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB81_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_xor_b64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB81_3
; GFX12-NEXT:    s_branch .LBB81_4
; GFX12-NEXT:  .LBB81_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB81_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_xor_b32_e32 v3, s5, v1
; GFX12-NEXT:    v_xor_b32_e32 v2, s4, v0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB81_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile xor ptr %ptr, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_load_i64_offset(ptr %in, ptr %out) {
; GCN1-LABEL: atomic_load_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %in, i64 4
  %val = load atomic i64, ptr %gep  seq_cst, align 8
  store i64 %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_load_i64(ptr %in, ptr %out) {
; GCN1-LABEL: atomic_load_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %val = load atomic i64, ptr %in syncscope("agent") seq_cst, align 8
  store i64 %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_load_i64_addr64_offset(ptr %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_load_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %in, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %val = load atomic i64, ptr %gep seq_cst, align 8
  store i64 %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_load_i64_addr64(ptr %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_load_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %in, i64 %index
  %val = load atomic i64, ptr %ptr seq_cst, align 8
  store i64 %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_store_i64_offset(i64 %in, ptr %out) {
; GCN1-LABEL: atomic_store_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_u32 s0, s2, 32
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    s_addc_u32 s1, s3, 0
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_u32 s0, s2, 32
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    s_addc_u32 s1, s3, 0
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  store atomic i64 %in, ptr %gep  seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_store_i64(i64 %in, ptr %out) {
; GCN1-LABEL: atomic_store_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  store atomic i64 %in, ptr %out seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_store_i64_addr64_offset(i64 %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_store_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN1-NEXT:    s_add_u32 s0, s2, s0
; GCN1-NEXT:    s_addc_u32 s1, s3, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN2-NEXT:    s_add_u32 s0, s2, s0
; GCN2-NEXT:    s_addc_u32 s1, s3, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  store atomic i64 %in, ptr %gep seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_store_i64_addr64(i64 %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_store_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN1-NEXT:    s_add_u32 s0, s2, s0
; GCN1-NEXT:    s_addc_u32 s1, s3, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN2-NEXT:    s_add_u32 s0, s2, s0
; GCN2-NEXT:    s_addc_u32 s1, s3, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  store atomic i64 %in, ptr %ptr seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_offset(ptr %out, i64 %in, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB90_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB90_4
; GCN1-NEXT:  .LBB90_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB90_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v5, s1
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s3
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    v_mov_b32_e32 v4, s0
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB90_2
; GCN1-NEXT:  .LBB90_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB90_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB90_4
; GCN2-NEXT:  .LBB90_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB90_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v5, s1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s3
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    v_mov_b32_e32 v4, s0
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB90_2
; GCN2-NEXT:  .LBB90_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB90_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB90_4
; GFX12-NEXT:  .LBB90_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB90_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB90_2
; GFX12-NEXT:  .LBB90_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_soffset(ptr %out, i64 %in, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_soffset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 0x11940
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB91_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB91_4
; GCN1-NEXT:  .LBB91_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB91_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v5, s1
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s3
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    v_mov_b32_e32 v4, s0
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB91_2
; GCN1-NEXT:  .LBB91_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_soffset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 0x11940
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB91_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB91_4
; GCN2-NEXT:  .LBB91_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB91_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v5, s1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s3
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    v_mov_b32_e32 v4, s0
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB91_2
; GCN2-NEXT:  .LBB91_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_soffset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 0x11940
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB91_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB91_4
; GFX12-NEXT:  .LBB91_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB91_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB91_2
; GFX12-NEXT:  .LBB91_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 9000
  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_ret_offset(ptr %out, ptr %out2, i64 %in, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s8, 32
; GCN1-NEXT:    s_addc_u32 s1, s9, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB92_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v5, s1
; GCN1-NEXT:    v_mov_b32_e32 v0, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s13
; GCN1-NEXT:    v_mov_b32_e32 v2, s14
; GCN1-NEXT:    v_mov_b32_e32 v3, s15
; GCN1-NEXT:    v_mov_b32_e32 v4, s0
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB92_3
; GCN1-NEXT:    s_branch .LBB92_4
; GCN1-NEXT:  .LBB92_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB92_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB92_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s8, 32
; GCN2-NEXT:    s_addc_u32 s1, s9, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB92_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v5, s1
; GCN2-NEXT:    v_mov_b32_e32 v0, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s13
; GCN2-NEXT:    v_mov_b32_e32 v2, s14
; GCN2-NEXT:    v_mov_b32_e32 v3, s15
; GCN2-NEXT:    v_mov_b32_e32 v4, s0
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB92_3
; GCN2-NEXT:    s_branch .LBB92_4
; GCN2-NEXT:  .LBB92_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB92_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB92_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s8, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s8
; GFX12-NEXT:    s_cbranch_vccz .LBB92_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
; GFX12-NEXT:    v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB92_3
; GFX12-NEXT:    s_branch .LBB92_4
; GFX12-NEXT:  .LBB92_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB92_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[6:7], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v1, s5, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v0, s4, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB92_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  %extract0 = extractvalue { i64, i1 } %val, 0
  store i64 %extract0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_addr64_offset(ptr %out, i64 %in, i64 %index, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[12:13], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB93_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB93_4
; GCN1-NEXT:  .LBB93_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB93_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v5, s1
; GCN1-NEXT:    v_mov_b32_e32 v0, s10
; GCN1-NEXT:    v_mov_b32_e32 v1, s11
; GCN1-NEXT:    v_mov_b32_e32 v2, s14
; GCN1-NEXT:    v_mov_b32_e32 v3, s15
; GCN1-NEXT:    v_mov_b32_e32 v4, s0
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB93_2
; GCN1-NEXT:  .LBB93_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s10
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[12:13], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB93_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB93_4
; GCN2-NEXT:  .LBB93_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB93_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v5, s1
; GCN2-NEXT:    v_mov_b32_e32 v0, s10
; GCN2-NEXT:    v_mov_b32_e32 v1, s11
; GCN2-NEXT:    v_mov_b32_e32 v2, s14
; GCN2-NEXT:    v_mov_b32_e32 v3, s15
; GCN2-NEXT:    v_mov_b32_e32 v4, s0
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB93_2
; GCN2-NEXT:  .LBB93_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s10
; GCN2-NEXT:    v_mov_b32_e32 v4, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[4:5], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB93_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB93_4
; GFX12-NEXT:  .LBB93_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB93_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB93_2
; GFX12-NEXT:  .LBB93_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[6:7], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x11
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x43
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[2:3], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s2, s8, s2
; GCN1-NEXT:    s_addc_u32 s3, s9, s3
; GCN1-NEXT:    s_add_u32 s2, s2, 32
; GCN1-NEXT:    s_addc_u32 s3, s3, 0
; GCN1-NEXT:    s_cmp_eq_u32 s3, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB94_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v5, s3
; GCN1-NEXT:    v_mov_b32_e32 v0, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s13
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v4, s2
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB94_3
; GCN1-NEXT:    s_branch .LBB94_4
; GCN1-NEXT:  .LBB94_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB94_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[2:3], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s2, s2, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    s_add_i32 s2, s2, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s2
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[0:1], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB94_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x44
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0x10c
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[2:3], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s2, s8, s2
; GCN2-NEXT:    s_addc_u32 s3, s9, s3
; GCN2-NEXT:    s_add_u32 s2, s2, 32
; GCN2-NEXT:    s_addc_u32 s3, s3, 0
; GCN2-NEXT:    s_cmp_eq_u32 s3, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB94_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v5, s3
; GCN2-NEXT:    v_mov_b32_e32 v0, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s13
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v4, s2
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB94_3
; GCN2-NEXT:    s_branch .LBB94_4
; GCN2-NEXT:  .LBB94_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB94_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[2:3], 0
; GCN2-NEXT:    s_cselect_b32 s2, s2, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    s_add_i32 s2, s2, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s2
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[0:1], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB94_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b256 s[8:15], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x44
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[2:3], s[14:15], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[8:9], s[2:3]
; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s3, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB94_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX12-NEXT:    v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX12-NEXT:    v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB94_3
; GFX12-NEXT:    s_branch .LBB94_4
; GFX12-NEXT:  .LBB94_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB94_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[2:3], 0
; GFX12-NEXT:    s_cselect_b32 s2, s2, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s2
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[0:1], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v1, s13, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v0, s12, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s2
; GFX12-NEXT:  .LBB94_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v3, s11
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  %extract0 = extractvalue { i64, i1 } %val, 0
  store i64 %extract0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64(ptr %out, i64 %in, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_mov_b64 s[6:7], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB95_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB95_4
; GCN1-NEXT:  .LBB95_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB95_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v5, s1
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s3
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    v_mov_b32_e32 v4, s0
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB95_2
; GCN1-NEXT:  .LBB95_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_mov_b64 s[6:7], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB95_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB95_4
; GCN2-NEXT:  .LBB95_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB95_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v5, s1
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s3
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    v_mov_b32_e32 v4, s0
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB95_2
; GCN2-NEXT:  .LBB95_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_mov_b32 s6, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB95_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB95_4
; GFX12-NEXT:  .LBB95_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB95_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB95_2
; GFX12-NEXT:  .LBB95_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %val = cmpxchg volatile ptr %out, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_ret(ptr %out, ptr %out2, i64 %in, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s0, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s9, s0
; GCN1-NEXT:    s_cselect_b64 s[0:1], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
; GCN1-NEXT:    s_cbranch_vccz .LBB96_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v4, s8
; GCN1-NEXT:    v_mov_b32_e32 v0, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s13
; GCN1-NEXT:    v_mov_b32_e32 v2, s14
; GCN1-NEXT:    v_mov_b32_e32 v3, s15
; GCN1-NEXT:    v_mov_b32_e32 v5, s9
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB96_3
; GCN1-NEXT:    s_branch .LBB96_4
; GCN1-NEXT:  .LBB96_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB96_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[0:1], s[8:9], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[0:1], s[0:1], exec
; GCN1-NEXT:    s_cselect_b32 s0, s8, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB96_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s0, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s9, s0
; GCN2-NEXT:    s_cselect_b64 s[0:1], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
; GCN2-NEXT:    s_cbranch_vccz .LBB96_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v4, s8
; GCN2-NEXT:    v_mov_b32_e32 v0, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s13
; GCN2-NEXT:    v_mov_b32_e32 v2, s14
; GCN2-NEXT:    v_mov_b32_e32 v3, s15
; GCN2-NEXT:    v_mov_b32_e32 v5, s9
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB96_3
; GCN2-NEXT:    s_branch .LBB96_4
; GCN2-NEXT:  .LBB96_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB96_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[8:9], 0
; GCN2-NEXT:    s_cselect_b32 s0, s8, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB96_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s8, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s8
; GFX12-NEXT:    s_cbranch_vccz .LBB96_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
; GFX12-NEXT:    v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB96_3
; GFX12-NEXT:    s_branch .LBB96_4
; GFX12-NEXT:  .LBB96_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB96_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[6:7], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v1, s5, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v0, s4, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB96_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %val = cmpxchg volatile ptr %out, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  %extract0 = extractvalue { i64, i1 } %val, 0
  store i64 %extract0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_addr64(ptr %out, i64 %in, i64 %index, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[12:13], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_mov_b64 s[2:3], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB97_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB97_4
; GCN1-NEXT:  .LBB97_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB97_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v5, s1
; GCN1-NEXT:    v_mov_b32_e32 v0, s10
; GCN1-NEXT:    v_mov_b32_e32 v1, s11
; GCN1-NEXT:    v_mov_b32_e32 v2, s14
; GCN1-NEXT:    v_mov_b32_e32 v3, s15
; GCN1-NEXT:    v_mov_b32_e32 v4, s0
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB97_2
; GCN1-NEXT:  .LBB97_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s10
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s11
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[12:13], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_mov_b64 s[2:3], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB97_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB97_4
; GCN2-NEXT:  .LBB97_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB97_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v5, s1
; GCN2-NEXT:    v_mov_b32_e32 v0, s10
; GCN2-NEXT:    v_mov_b32_e32 v1, s11
; GCN2-NEXT:    v_mov_b32_e32 v2, s14
; GCN2-NEXT:    v_mov_b32_e32 v3, s15
; GCN2-NEXT:    v_mov_b32_e32 v4, s0
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB97_2
; GCN2-NEXT:  .LBB97_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s10
; GCN2-NEXT:    v_mov_b32_e32 v4, s11
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[14:15], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[4:5], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB97_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB97_4
; GFX12-NEXT:  .LBB97_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB97_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT:    v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[4:5], v[0:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB97_2
; GFX12-NEXT:  .LBB97_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[6:7], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %val = cmpxchg volatile ptr %ptr, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  ret void
}

define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index, i64 %old) {
; GCN1-LABEL: atomic_cmpxchg_i64_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x43
; GCN1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x11
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[2:3], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s2, s8, s2
; GCN1-NEXT:    s_addc_u32 s3, s9, s3
; GCN1-NEXT:    s_cmp_eq_u32 s3, s6
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB98_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v5, s3
; GCN1-NEXT:    v_mov_b32_e32 v0, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s13
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v4, s2
; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB98_3
; GCN1-NEXT:    s_branch .LBB98_4
; GCN1-NEXT:  .LBB98_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB98_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[2:3], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s2, s2, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    s_add_i32 s2, s2, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s2
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, s[0:1], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB98_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i64_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0x10c
; GCN2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x44
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[2:3], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s2, s8, s2
; GCN2-NEXT:    s_addc_u32 s3, s9, s3
; GCN2-NEXT:    s_cmp_eq_u32 s3, s6
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB98_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v5, s3
; GCN2-NEXT:    v_mov_b32_e32 v0, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s13
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v4, s2
; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB98_3
; GCN2-NEXT:    s_branch .LBB98_4
; GCN2-NEXT:  .LBB98_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB98_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[2:3], 0
; GCN2-NEXT:    s_cselect_b32 s2, s2, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    s_add_i32 s2, s2, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s2
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, s[0:1], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB98_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_cmpxchg_i64_ret_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b256 s[8:15], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x44
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[2:3], s[14:15], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[8:9], s[2:3]
; GFX12-NEXT:    s_cmp_eq_u32 s3, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB98_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX12-NEXT:    v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX12-NEXT:    v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
; GFX12-NEXT:    flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB98_3
; GFX12-NEXT:    s_branch .LBB98_4
; GFX12-NEXT:  .LBB98_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB98_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[2:3], 0
; GFX12-NEXT:    s_cselect_b32 s2, s2, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s2
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[0:1], v[0:1]
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v1, s13, vcc_lo
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v0, s12, vcc_lo
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s2
; GFX12-NEXT:  .LBB98_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v3, s11
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %val = cmpxchg volatile ptr %ptr, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst
  %extract0 = extractvalue { i64, i1 } %val, 0
  store i64 %extract0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_load_f64_offset(ptr %in, ptr %out) {
; GCN1-LABEL: atomic_load_f64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_f64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr double, ptr %in, i64 4
  %val = load atomic double, ptr %gep  seq_cst, align 8
  store double %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_load_f64(ptr %in, ptr %out) {
; GCN1-LABEL: atomic_load_f64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_f64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %val = load atomic double, ptr %in syncscope("agent") seq_cst, align 8
  store double %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_load_f64_addr64_offset(ptr %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_load_f64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_f64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr double, ptr %in, i64 %index
  %gep = getelementptr double, ptr %ptr, i64 4
  %val = load atomic double, ptr %gep seq_cst, align 8
  store double %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_load_f64_addr64(ptr %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_load_f64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_load_f64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_load_f64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr double, ptr %in, i64 %index
  %val = load atomic double, ptr %ptr seq_cst, align 8
  store double %val, ptr %out
  ret void
}

define amdgpu_kernel void @atomic_store_f64_offset(double %in, ptr %out) {
; GCN1-LABEL: atomic_store_f64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    s_add_u32 s0, s2, 32
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    s_addc_u32 s1, s3, 0
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_f64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    s_add_u32 s0, s2, 32
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    s_addc_u32 s1, s3, 0
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr double, ptr %out, i64 4
  store atomic double %in, ptr %gep  seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_store_f64(double %in, ptr %out) {
; GCN1-LABEL: atomic_store_f64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_f64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  store atomic double %in, ptr %out seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_store_f64_addr64_offset(double %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_store_f64_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN1-NEXT:    s_add_u32 s0, s2, s0
; GCN1-NEXT:    s_addc_u32 s1, s3, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_f64_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN2-NEXT:    s_add_u32 s0, s2, s0
; GCN2-NEXT:    s_addc_u32 s1, s3, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f64_addr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] offset:32 scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr double, ptr %out, i64 %index
  %gep = getelementptr double, ptr %ptr, i64 4
  store atomic double %in, ptr %gep seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_store_f64_addr64(double %in, ptr %out, i64 %index) {
; GCN1-LABEL: atomic_store_f64_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN1-NEXT:    s_add_u32 s0, s2, s0
; GCN1-NEXT:    s_addc_u32 s1, s3, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_store_f64_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[4:5], 3
; GCN2-NEXT:    s_add_u32 s0, s2, s0
; GCN2-NEXT:    s_addc_u32 s1, s3, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_store_f64_addr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    global_wb scope:SCOPE_SYS
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1] scope:SCOPE_SYS
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr double, ptr %out, i64 %index
  store atomic double %in, ptr %ptr seq_cst, align 8
  ret void
}

define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_inc_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB107_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB107_4
; GCN1-NEXT:  .LBB107_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB107_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB107_2
; GCN1-NEXT:  .LBB107_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v1, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v0, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB107_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB107_4
; GCN2-NEXT:  .LBB107_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB107_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB107_2
; GCN2-NEXT:  .LBB107_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v1, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v0, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB107_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB107_4
; GFX12-NEXT:  .LBB107_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB107_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB107_2
; GFX12-NEXT:  .LBB107_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    s_wait_alu 0xfffd
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT:    v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_inc_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_inc_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB108_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB108_3
; GCN1-NEXT:    s_branch .LBB108_4
; GCN1-NEXT:  .LBB108_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB108_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB108_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB108_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB108_3
; GCN2-NEXT:    s_branch .LBB108_4
; GCN2-NEXT:  .LBB108_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB108_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB108_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB108_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB108_3
; GFX12-NEXT:    s_branch .LBB108_4
; GFX12-NEXT:  .LBB108_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB108_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    s_wait_alu 0xfffd
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT:    v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB108_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_inc_i64_incr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB109_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB109_4
; GCN1-NEXT:  .LBB109_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB109_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB109_2
; GCN1-NEXT:  .LBB109_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v1, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v0, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64_incr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB109_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB109_4
; GCN2-NEXT:  .LBB109_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB109_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB109_2
; GCN2-NEXT:  .LBB109_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v1, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v0, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_incr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB109_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB109_4
; GFX12-NEXT:  .LBB109_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB109_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB109_2
; GFX12-NEXT:  .LBB109_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    s_wait_alu 0xfffd
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT:    v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_inc_i64_ret_incr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_inc_i64_ret_incr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB110_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB110_3
; GCN1-NEXT:    s_branch .LBB110_4
; GCN1-NEXT:  .LBB110_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB110_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB110_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64_ret_incr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB110_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB110_3
; GCN2-NEXT:    s_branch .LBB110_4
; GCN2-NEXT:  .LBB110_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB110_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB110_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_ret_incr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB110_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB110_3
; GFX12-NEXT:    s_branch .LBB110_4
; GFX12-NEXT:  .LBB110_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB110_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    s_wait_alu 0xfffd
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT:    v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB110_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_inc_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_inc_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB111_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB111_4
; GCN1-NEXT:  .LBB111_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB111_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB111_2
; GCN1-NEXT:  .LBB111_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v1, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v0, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB111_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB111_4
; GCN2-NEXT:  .LBB111_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB111_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB111_2
; GCN2-NEXT:  .LBB111_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v1, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v0, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB111_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB111_4
; GFX12-NEXT:  .LBB111_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB111_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB111_2
; GFX12-NEXT:  .LBB111_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    s_wait_alu 0xfffd
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT:    v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile uinc_wrap ptr %out, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_inc_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_inc_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s6
; GCN1-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_vccz .LBB112_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB112_3
; GCN1-NEXT:    s_branch .LBB112_4
; GCN1-NEXT:  .LBB112_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB112_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[6:7], s[6:7], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v4, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB112_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s6
; GCN2-NEXT:    s_cselect_b64 s[6:7], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_vccz .LBB112_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB112_3
; GCN2-NEXT:    s_branch .LBB112_4
; GCN2-NEXT:  .LBB112_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB112_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB112_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB112_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB112_3
; GFX12-NEXT:    s_branch .LBB112_4
; GFX12-NEXT:  .LBB112_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB112_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    s_wait_alu 0xfffd
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT:    v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB112_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile uinc_wrap ptr %out, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_inc_i64_incr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB113_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB113_4
; GCN1-NEXT:  .LBB113_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB113_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB113_2
; GCN1-NEXT:  .LBB113_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v1, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v0, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64_incr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB113_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB113_4
; GCN2-NEXT:  .LBB113_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB113_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB113_2
; GCN2-NEXT:  .LBB113_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v0, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v1, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v0, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_incr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB113_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB113_4
; GFX12-NEXT:  .LBB113_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB113_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB113_2
; GFX12-NEXT:  .LBB113_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[0:1]
; GFX12-NEXT:    s_wait_alu 0xfffd
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT:    v_dual_cndmask_b32 v1, 0, v3 :: v_dual_cndmask_b32 v0, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile uinc_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_inc_i64_ret_incr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_inc_i64_ret_incr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB114_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB114_3
; GCN1-NEXT:    s_branch .LBB114_4
; GCN1-NEXT:  .LBB114_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB114_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[12:13], v[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN1-NEXT:    buffer_store_dword v4, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v5, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB114_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_inc_i64_ret_incr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB114_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB114_3
; GCN2-NEXT:    s_branch .LBB114_4
; GCN2-NEXT:  .LBB114_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB114_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[12:13], v[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
; GCN2-NEXT:    buffer_store_dword v4, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v5, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB114_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_ret_incr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB114_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_inc_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB114_3
; GFX12-NEXT:    s_branch .LBB114_4
; GFX12-NEXT:  .LBB114_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB114_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s0, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s0
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, 1
; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v1, vcc_lo
; GFX12-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[4:5], v[0:1]
; GFX12-NEXT:    s_wait_alu 0xfffd
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT:    v_dual_cndmask_b32 v3, 0, v3 :: v_dual_cndmask_b32 v2, 0, v2
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s0
; GFX12-NEXT:  .LBB114_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile uinc_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_dec_i64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB115_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB115_4
; GCN1-NEXT:  .LBB115_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB115_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB115_2
; GCN1-NEXT:  .LBB115_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]
; GCN1-NEXT:    v_add_i32_e64 v0, s[2:3], -1, v0
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN1-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB115_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB115_4
; GCN2-NEXT:  .LBB115_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB115_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB115_2
; GCN2-NEXT:  .LBB115_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]
; GCN2-NEXT:    v_add_u32_e64 v0, s[2:3], -1, v0
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN2-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB115_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB115_4
; GFX12-NEXT:  .LBB115_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB115_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB115_2
; GFX12-NEXT:  .LBB115_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s4, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s4
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[2:3], v[0:1]
; GFX12-NEXT:    v_add_co_u32 v0, s1, v0, -1
; GFX12-NEXT:    s_wait_alu 0xf1ff
; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, s1, -1, v1, s1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, s0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s4
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_dec_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_dec_i64_ret_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s6, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s2, s8, 32
; GCN1-NEXT:    s_addc_u32 s3, s9, 0
; GCN1-NEXT:    s_cmp_eq_u32 s3, s6
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB116_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v1, s3
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB116_3
; GCN1-NEXT:    s_branch .LBB116_4
; GCN1-NEXT:  .LBB116_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB116_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[2:3], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s1
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s2, s2, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    s_add_i32 s2, s2, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s2
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v5, s0
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e64 v6, s[2:3], -1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[0:1], v[0:1]
; GCN1-NEXT:    v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB116_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64_ret_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s6, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s2, s8, 32
; GCN2-NEXT:    s_addc_u32 s3, s9, 0
; GCN2-NEXT:    s_cmp_eq_u32 s3, s6
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB116_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v1, s3
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB116_3
; GCN2-NEXT:    s_branch .LBB116_4
; GCN2-NEXT:  .LBB116_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB116_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[2:3], 0
; GCN2-NEXT:    s_cselect_b32 s2, s2, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    s_add_i32 s2, s2, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s2
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s1
; GCN2-NEXT:    v_mov_b32_e32 v5, s0
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e64 v6, s[2:3], -1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[0:1], v[0:1]
; GCN2-NEXT:    v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB116_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_ret_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB116_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB116_3
; GFX12-NEXT:    s_branch .LBB116_4
; GFX12-NEXT:  .LBB116_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB116_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s6, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s6
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[4:5], v[0:1]
; GFX12-NEXT:    v_add_co_u32 v2, s1, v0, -1
; GFX12-NEXT:    s_wait_alu 0xf1ff
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, s1, -1, v1, s1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v3, s5, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v2, s4, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s6
; GFX12-NEXT:  .LBB116_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %gep = getelementptr i64, ptr %out, i64 4
  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_dec_i64_decr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB117_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB117_4
; GCN1-NEXT:  .LBB117_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB117_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB117_2
; GCN1-NEXT:  .LBB117_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]
; GCN1-NEXT:    v_add_i32_e64 v0, s[2:3], -1, v0
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN1-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64_decr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB117_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB117_4
; GCN2-NEXT:  .LBB117_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB117_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB117_2
; GCN2-NEXT:  .LBB117_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]
; GCN2-NEXT:    v_add_u32_e64 v0, s[2:3], -1, v0
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN2-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_decr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB117_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB117_4
; GFX12-NEXT:  .LBB117_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB117_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB117_2
; GFX12-NEXT:  .LBB117_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s4, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s4
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[2:3], v[0:1]
; GFX12-NEXT:    v_add_co_u32 v0, s1, v0, -1
; GFX12-NEXT:    s_wait_alu 0xf1ff
; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, s1, -1, v1, s1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, s0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s4
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_dec_i64_ret_decr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_dec_i64_ret_decr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_add_u32 s0, s0, 32
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB118_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB118_3
; GCN1-NEXT:    s_branch .LBB118_4
; GCN1-NEXT:  .LBB118_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB118_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e64 v6, s[2:3], -1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[12:13], v[0:1]
; GCN1-NEXT:    v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB118_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64_ret_decr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_add_u32 s0, s0, 32
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB118_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB118_3
; GCN2-NEXT:    s_branch .LBB118_4
; GCN2-NEXT:  .LBB118_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB118_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e64 v6, s[2:3], -1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[12:13], v[0:1]
; GCN2-NEXT:    v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB118_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_ret_decr64_offset:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 32
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB118_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB118_3
; GFX12-NEXT:    s_branch .LBB118_4
; GFX12-NEXT:  .LBB118_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB118_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s6, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s6
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[4:5], v[0:1]
; GFX12-NEXT:    v_add_co_u32 v2, s1, v0, -1
; GFX12-NEXT:    s_wait_alu 0xf1ff
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, s1, -1, v1, s1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v3, s5, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v2, s4, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s6
; GFX12-NEXT:  .LBB118_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %gep = getelementptr i64, ptr %ptr, i64 4
  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_dec_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_dec_i64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x3d
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s1, s4
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB119_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB119_4
; GCN1-NEXT:  .LBB119_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB119_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB119_2
; GCN1-NEXT:  .LBB119_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]
; GCN1-NEXT:    v_add_i32_e64 v0, s[2:3], -1, v0
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN1-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s4, s[4:5], 0xf4
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s1, s4
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB119_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB119_4
; GCN2-NEXT:  .LBB119_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB119_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB119_2
; GCN2-NEXT:  .LBB119_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]
; GCN2-NEXT:    v_add_u32_e64 v0, s[2:3], -1, v0
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN2-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[4:5], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s5
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB119_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB119_4
; GFX12-NEXT:  .LBB119_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB119_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB119_2
; GFX12-NEXT:  .LBB119_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s4, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s4
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[2:3], v[0:1]
; GFX12-NEXT:    v_add_co_u32 v0, s1, v0, -1
; GFX12-NEXT:    s_wait_alu 0xf1ff
; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, s1, -1, v1, s1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, s0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s4
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile udec_wrap ptr %out, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_dec_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-LABEL: atomic_dec_i64_ret:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x3f
; GCN1-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
; GCN1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_cmp_eq_u32 s9, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB120_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s8
; GCN1-NEXT:    v_mov_b32_e32 v3, s1
; GCN1-NEXT:    v_mov_b32_e32 v1, s9
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB120_3
; GCN1-NEXT:    s_branch .LBB120_4
; GCN1-NEXT:  .LBB120_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB120_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[8:9], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s1
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s2, s8, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    s_add_i32 s2, s2, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s2
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v5, s0
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e64 v6, s[2:3], -1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[0:1], v[0:1]
; GCN1-NEXT:    v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[12:15], 0 offen
; GCN1-NEXT:  .LBB120_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64_ret:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0xfc
; GCN2-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x24
; GCN2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_cmp_eq_u32 s9, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB120_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s8
; GCN2-NEXT:    v_mov_b32_e32 v3, s1
; GCN2-NEXT:    v_mov_b32_e32 v1, s9
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB120_3
; GCN2-NEXT:    s_branch .LBB120_4
; GCN2-NEXT:  .LBB120_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB120_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[8:9], 0
; GCN2-NEXT:    s_cselect_b32 s2, s8, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    s_add_i32 s2, s2, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s2
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s1
; GCN2-NEXT:    v_mov_b32_e32 v5, s0
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e64 v6, s[2:3], -1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[0:1], v[0:1]
; GCN2-NEXT:    v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB120_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_ret:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB120_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB120_3
; GFX12-NEXT:    s_branch .LBB120_4
; GFX12-NEXT:  .LBB120_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB120_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s6, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s6
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[4:5], v[0:1]
; GFX12-NEXT:    v_add_co_u32 v2, s1, v0, -1
; GFX12-NEXT:    s_wait_alu 0xf1ff
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, s1, -1, v1, s1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v3, s5, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v2, s4, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s6
; GFX12-NEXT:  .LBB120_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw volatile udec_wrap ptr %out, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_dec_i64_decr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s14, -1
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s8, s[4:5], 0x3f
; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
; GCN1-NEXT:    s_add_u32 s12, s12, s11
; GCN1-NEXT:    s_addc_u32 s13, s13, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_cmp_eq_u32 s1, s8
; GCN1-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_mov_b64 s[4:5], -1
; GCN1-NEXT:    s_cbranch_vccnz .LBB121_3
; GCN1-NEXT:  ; %bb.1: ; %Flow
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_vccz .LBB121_4
; GCN1-NEXT:  .LBB121_2: ; %atomicrmw.phi
; GCN1-NEXT:    s_endpgm
; GCN1-NEXT:  .LBB121_3: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s3
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execnz .LBB121_2
; GCN1-NEXT:  .LBB121_4: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s3
; GCN1-NEXT:    s_and_b64 s[4:5], s[4:5], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v5, s2
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]
; GCN1-NEXT:    v_add_i32_e64 v0, s[2:3], -1, v0
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN1-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN1-NEXT:    buffer_store_dword v0, v2, s[12:15], 0 offen
; GCN1-NEXT:    buffer_store_dword v1, v3, s[12:15], 0 offen
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64_decr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s8, s[4:5], 0xfc
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_cmp_eq_u32 s1, s8
; GCN2-NEXT:    s_cselect_b64 s[4:5], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_mov_b64 s[4:5], -1
; GCN2-NEXT:    s_cbranch_vccnz .LBB121_3
; GCN2-NEXT:  ; %bb.1: ; %Flow
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_vccz .LBB121_4
; GCN2-NEXT:  .LBB121_2: ; %atomicrmw.phi
; GCN2-NEXT:    s_endpgm
; GCN2-NEXT:  .LBB121_3: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s3
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execnz .LBB121_2
; GCN2-NEXT:  .LBB121_4: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v4, s3
; GCN2-NEXT:    v_mov_b32_e32 v5, s2
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[2:3], v[0:1]
; GCN2-NEXT:    v_add_u32_e64 v0, s[2:3], -1, v0
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN2-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v1, s[2:3]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
; GCN2-NEXT:    buffer_store_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_decr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_clause 0x1
; GFX12-NEXT:    s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
; GFX12-NEXT:    s_mov_b64 s[6:7], src_private_base
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_cmp_eq_u32 s1, s7
; GFX12-NEXT:    s_cselect_b32 s4, -1, 0
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_mov_b32 s4, -1
; GFX12-NEXT:    s_cbranch_vccnz .LBB121_3
; GFX12-NEXT:  ; %bb.1: ; %Flow
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT:    s_cbranch_vccz .LBB121_4
; GFX12-NEXT:  .LBB121_2: ; %atomicrmw.phi
; GFX12-NEXT:    s_endpgm
; GFX12-NEXT:  .LBB121_3: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execnz .LBB121_2
; GFX12-NEXT:  .LBB121_4: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s4, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s4
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[2:3], v[0:1]
; GFX12-NEXT:    v_add_co_u32 v0, s1, v0, -1
; GFX12-NEXT:    s_wait_alu 0xf1ff
; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, s1, -1, v1, s1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    v_cndmask_b32_e64 v1, v1, s3, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, s0
; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s4
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile udec_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @atomic_dec_i64_ret_decr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_dec_i64_ret_decr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
; GCN1-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
; GCN1-NEXT:    s_mov_b32 s18, -1
; GCN1-NEXT:    s_mov_b32 s19, 0xe8f000
; GCN1-NEXT:    s_add_u32 s16, s16, s11
; GCN1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0x41
; GCN1-NEXT:    s_addc_u32 s17, s17, 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN1-NEXT:    s_add_u32 s0, s8, s0
; GCN1-NEXT:    s_addc_u32 s1, s9, s1
; GCN1-NEXT:    s_cmp_eq_u32 s1, s2
; GCN1-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT:    s_cbranch_vccz .LBB122_2
; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v2, s12
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    v_mov_b32_e32 v3, s13
; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_cbranch_execz .LBB122_3
; GCN1-NEXT:    s_branch .LBB122_4
; GCN1-NEXT:  .LBB122_2:
; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT:  .LBB122_3: ; %atomicrmw.private
; GCN1-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GCN1-NEXT:    v_mov_b32_e32 v5, s12
; GCN1-NEXT:    s_and_b64 s[2:3], s[2:3], exec
; GCN1-NEXT:    s_cselect_b32 s0, s0, -1
; GCN1-NEXT:    v_mov_b32_e32 v2, s0
; GCN1-NEXT:    s_add_i32 s0, s0, 4
; GCN1-NEXT:    v_mov_b32_e32 v3, s0
; GCN1-NEXT:    buffer_load_dword v0, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_load_dword v1, v3, s[16:19], 0 offen
; GCN1-NEXT:    v_mov_b32_e32 v4, s13
; GCN1-NEXT:    s_waitcnt vmcnt(1)
; GCN1-NEXT:    v_add_i32_e64 v6, s[2:3], -1, v0
; GCN1-NEXT:    s_waitcnt vmcnt(0)
; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[12:13], v[0:1]
; GCN1-NEXT:    v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN1-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN1-NEXT:    buffer_store_dword v5, v2, s[16:19], 0 offen
; GCN1-NEXT:    buffer_store_dword v4, v3, s[16:19], 0 offen
; GCN1-NEXT:  .LBB122_4: ; %atomicrmw.end
; GCN1-NEXT:    v_mov_b32_e32 v2, s10
; GCN1-NEXT:    v_mov_b32_e32 v3, s11
; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_dec_i64_ret_decr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; GCN2-NEXT:    s_mov_b32 s90, -1
; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
; GCN2-NEXT:    s_add_u32 s88, s88, s11
; GCN2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT:    s_addc_u32 s89, s89, 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT:    s_add_u32 s0, s8, s0
; GCN2-NEXT:    s_addc_u32 s1, s9, s1
; GCN2-NEXT:    s_cmp_eq_u32 s1, s2
; GCN2-NEXT:    s_cselect_b64 s[2:3], -1, 0
; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT:    s_cbranch_vccz .LBB122_2
; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v2, s12
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    v_mov_b32_e32 v3, s13
; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_cbranch_execz .LBB122_3
; GCN2-NEXT:    s_branch .LBB122_4
; GCN2-NEXT:  .LBB122_2:
; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT:  .LBB122_3: ; %atomicrmw.private
; GCN2-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GCN2-NEXT:    s_cselect_b32 s0, s0, -1
; GCN2-NEXT:    v_mov_b32_e32 v2, s0
; GCN2-NEXT:    s_add_i32 s0, s0, 4
; GCN2-NEXT:    v_mov_b32_e32 v3, s0
; GCN2-NEXT:    buffer_load_dword v0, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_load_dword v1, v3, s[88:91], 0 offen
; GCN2-NEXT:    v_mov_b32_e32 v5, s12
; GCN2-NEXT:    v_mov_b32_e32 v4, s13
; GCN2-NEXT:    s_waitcnt vmcnt(1)
; GCN2-NEXT:    v_add_u32_e64 v6, s[2:3], -1, v0
; GCN2-NEXT:    s_waitcnt vmcnt(0)
; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[12:13], v[0:1]
; GCN2-NEXT:    v_addc_u32_e64 v7, s[2:3], -1, v1, s[2:3]
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[0:1]
; GCN2-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
; GCN2-NEXT:    buffer_store_dword v5, v2, s[88:91], 0 offen
; GCN2-NEXT:    buffer_store_dword v4, v3, s[88:91], 0 offen
; GCN2-NEXT:  .LBB122_4: ; %atomicrmw.end
; GCN2-NEXT:    v_mov_b32_e32 v2, s10
; GCN2-NEXT:    v_mov_b32_e32 v3, s11
; GCN2-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT:    s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_ret_decr64:
; GFX12:       ; %bb.0: ; %entry
; GFX12-NEXT:    s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT:    s_mov_b64 s[8:9], src_private_base
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT:    s_cmp_eq_u32 s1, s9
; GFX12-NEXT:    s_cselect_b32 s6, -1, 0
; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT:    s_cbranch_vccz .LBB122_2
; GFX12-NEXT:  ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT:    flat_atomic_dec_u64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT:    global_inv scope:SCOPE_DEV
; GFX12-NEXT:    s_cbranch_execz .LBB122_3
; GFX12-NEXT:    s_branch .LBB122_4
; GFX12-NEXT:  .LBB122_2:
; GFX12-NEXT:    ; implicit-def: $vgpr0_vgpr1
; GFX12-NEXT:  .LBB122_3: ; %atomicrmw.private
; GFX12-NEXT:    s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT:    s_cselect_b32 s6, s0, -1
; GFX12-NEXT:    scratch_load_b64 v[0:1], off, s6
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT:    v_cmp_lt_u64_e64 s0, s[4:5], v[0:1]
; GFX12-NEXT:    v_add_co_u32 v2, s1, v0, -1
; GFX12-NEXT:    s_wait_alu 0xf1ff
; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, s1, -1, v1, s1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-NEXT:    s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT:    s_wait_alu 0xfffe
; GFX12-NEXT:    v_cndmask_b32_e64 v3, v3, s5, s0
; GFX12-NEXT:    v_cndmask_b32_e64 v2, v2, s4, s0
; GFX12-NEXT:    scratch_store_b64 off, v[2:3], s6
; GFX12-NEXT:  .LBB122_4: ; %atomicrmw.end
; GFX12-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
; GFX12-NEXT:    s_endpgm
entry:
  %ptr = getelementptr i64, ptr %out, i64 %index
  %tmp0 = atomicrmw volatile udec_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst
  store i64 %tmp0, ptr %out2
  ret void
}