Google Git
Sign in
llvm/llvm-project/5a6cc509215b62e94de3b798ea26944a375ce6cb/./llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
blob: d7bd4b1e4918e851b23e504ffd7e1b99a3231fc9 [file]
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=GCN1 %s
; RUN: llc -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN2 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN3 %s
; ---------------------------------------------------------------------
; atomicrmw xchg
; ---------------------------------------------------------------------
define void @flat_atomic_xchg_i64_noret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_xchg_i64_noret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB0_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB0_4
; GCN1-NEXT: .LBB0_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB0_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB0_2
; GCN1-NEXT: .LBB0_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i64_noret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB0_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB0_4
; GCN2-NEXT: .LBB0_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB0_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB0_2
; GCN2-NEXT: .LBB0_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i64_noret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB0_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB0_4
; GCN3-NEXT: .LBB0_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB0_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB0_2
; GCN3-NEXT: .LBB0_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw xchg ptr %ptr, i64 %in seq_cst
ret void
}
define void @flat_atomic_xchg_i64_noret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB1_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB1_4
; GCN1-NEXT: .LBB1_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB1_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB1_2
; GCN1-NEXT: .LBB1_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB1_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB1_4
; GCN2-NEXT: .LBB1_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB1_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB1_2
; GCN2-NEXT: .LBB1_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB1_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB1_4
; GCN3-NEXT: .LBB1_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB1_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB1_2
; GCN3-NEXT: .LBB1_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst
ret void
}
define i64 @flat_atomic_xchg_i64_ret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_xchg_i64_ret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_mov_b32_e32 v5, v1
; GCN1-NEXT: v_mov_b32_e32 v4, v0
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB2_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB2_4
; GCN1-NEXT: .LBB2_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB2_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB2_2
; GCN1-NEXT: .LBB2_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i64_ret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_mov_b32_e32 v5, v1
; GCN2-NEXT: v_mov_b32_e32 v4, v0
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB2_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB2_4
; GCN2-NEXT: .LBB2_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB2_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB2_2
; GCN2-NEXT: .LBB2_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v5, v1
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_mov_b32_e32 v4, v0
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB2_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB2_4
; GCN3-NEXT: .LBB2_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB2_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB2_2
; GCN3-NEXT: .LBB2_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_nop 0
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw xchg ptr %ptr, i64 %in seq_cst
ret i64 %result
}
define i64 @flat_atomic_xchg_i64_ret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB3_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB3_4
; GCN1-NEXT: .LBB3_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB3_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB3_2
; GCN1-NEXT: .LBB3_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB3_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB3_4
; GCN2-NEXT: .LBB3_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB3_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB3_2
; GCN2-NEXT: .LBB3_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB3_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB3_4
; GCN3-NEXT: .LBB3_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB3_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB3_2
; GCN3-NEXT: .LBB3_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_nop 0
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw xchg ptr %gep, i64 %in seq_cst
ret i64 %result
}
define amdgpu_gfx void @flat_atomic_xchg_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_i64_noret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_mov_b64 s[34:35], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB4_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB4_4
; GCN1-NEXT: .LBB4_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB4_3: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execnz .LBB4_2
; GCN1-NEXT: .LBB4_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: v_mov_b32_e32 v0, s6
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v0, s7
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i64_noret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_mov_b64 s[34:35], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB4_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB4_4
; GCN2-NEXT: .LBB4_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB4_3: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execnz .LBB4_2
; GCN2-NEXT: .LBB4_4: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v0, s7
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i64_noret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_mov_b64 s[34:35], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB4_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB4_4
; GCN3-NEXT: .LBB4_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB4_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execnz .LBB4_2
; GCN3-NEXT: .LBB4_4: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s7
; GCN3-NEXT: v_mov_b32_e32 v1, s34
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw xchg ptr %ptr, i64 %in seq_cst
ret void
}
define amdgpu_gfx void @flat_atomic_xchg_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_mov_b64 s[36:37], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB5_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB5_4
; GCN1-NEXT: .LBB5_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB5_3: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execnz .LBB5_2
; GCN1-NEXT: .LBB5_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v0, s6
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v0, s7
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_mov_b64 s[36:37], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB5_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB5_4
; GCN2-NEXT: .LBB5_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB5_3: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execnz .LBB5_2
; GCN2-NEXT: .LBB5_4: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v0, s7
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_mov_b64 s[36:37], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB5_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB5_4
; GCN3-NEXT: .LBB5_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB5_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s35
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execnz .LBB5_2
; GCN3-NEXT: .LBB5_4: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s7
; GCN3-NEXT: v_mov_b32_e32 v1, s34
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst
ret void
}
define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_i64_ret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB6_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execz .LBB6_3
; GCN1-NEXT: s_branch .LBB6_4
; GCN1-NEXT: .LBB6_2:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: .LBB6_3: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s6
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v2, s7
; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB6_4: ; %atomicrmw.end
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i64_ret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB6_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execz .LBB6_3
; GCN2-NEXT: s_branch .LBB6_4
; GCN2-NEXT: .LBB6_2:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: .LBB6_3: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v4, s6
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v2, s7
; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB6_4: ; %atomicrmw.end
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i64_ret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB6_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execz .LBB6_3
; GCN3-NEXT: s_branch .LBB6_4
; GCN3-NEXT: .LBB6_2:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: .LBB6_3: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v3, s6
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB6_4: ; %atomicrmw.end
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw xchg ptr %ptr, i64 %in seq_cst
ret i64 %result
}
define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB7_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execz .LBB7_3
; GCN1-NEXT: s_branch .LBB7_4
; GCN1-NEXT: .LBB7_2:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: .LBB7_3: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s6
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v2, s7
; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB7_4: ; %atomicrmw.end
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB7_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execz .LBB7_3
; GCN2-NEXT: s_branch .LBB7_4
; GCN2-NEXT: .LBB7_2:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: .LBB7_3: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v4, s6
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v2, s7
; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB7_4: ; %atomicrmw.end
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB7_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s35
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execz .LBB7_3
; GCN3-NEXT: s_branch .LBB7_4
; GCN3-NEXT: .LBB7_2:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: .LBB7_3: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v3, s6
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB7_4: ; %atomicrmw.end
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw xchg ptr %gep, i64 %in seq_cst
ret i64 %result
}
define void @flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB8_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB8_4
; GCN1-NEXT: .LBB8_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB8_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB8_2
; GCN1-NEXT: .LBB8_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB8_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB8_4
; GCN2-NEXT: .LBB8_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB8_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB8_2
; GCN2-NEXT: .LBB8_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB8_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB8_4
; GCN3-NEXT: .LBB8_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB8_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB8_2
; GCN3-NEXT: .LBB8_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
ret void
}
define i64 @flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB9_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB9_4
; GCN1-NEXT: .LBB9_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB9_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB9_2
; GCN1-NEXT: .LBB9_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB9_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB9_4
; GCN2-NEXT: .LBB9_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB9_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB9_2
; GCN2-NEXT: .LBB9_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB9_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB9_4
; GCN3-NEXT: .LBB9_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB9_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB9_2
; GCN3-NEXT: .LBB9_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_nop 0
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw xchg ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
ret i64 %result
}
; ---------------------------------------------------------------------
; atomicrmw xchg f64
; ---------------------------------------------------------------------
define void @flat_atomic_xchg_f64_noret(ptr %ptr, double %in) {
; GCN1-LABEL: flat_atomic_xchg_f64_noret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB10_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB10_4
; GCN1-NEXT: .LBB10_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB10_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB10_2
; GCN1-NEXT: .LBB10_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f64_noret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB10_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB10_4
; GCN2-NEXT: .LBB10_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB10_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB10_2
; GCN2-NEXT: .LBB10_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f64_noret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB10_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB10_4
; GCN3-NEXT: .LBB10_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB10_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB10_2
; GCN3-NEXT: .LBB10_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw xchg ptr %ptr, double %in seq_cst
ret void
}
define void @flat_atomic_xchg_f64_noret_offset(ptr %out, double %in) {
; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB11_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB11_4
; GCN1-NEXT: .LBB11_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB11_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB11_2
; GCN1-NEXT: .LBB11_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB11_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB11_4
; GCN2-NEXT: .LBB11_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB11_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB11_2
; GCN2-NEXT: .LBB11_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB11_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB11_4
; GCN3-NEXT: .LBB11_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB11_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB11_2
; GCN3-NEXT: .LBB11_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr %out, i32 4
%tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst
ret void
}
define double @flat_atomic_xchg_f64_ret(ptr %ptr, double %in) {
; GCN1-LABEL: flat_atomic_xchg_f64_ret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_mov_b32_e32 v5, v1
; GCN1-NEXT: v_mov_b32_e32 v4, v0
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB12_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB12_4
; GCN1-NEXT: .LBB12_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB12_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB12_2
; GCN1-NEXT: .LBB12_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f64_ret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_mov_b32_e32 v5, v1
; GCN2-NEXT: v_mov_b32_e32 v4, v0
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB12_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB12_4
; GCN2-NEXT: .LBB12_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB12_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB12_2
; GCN2-NEXT: .LBB12_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v5, v1
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_mov_b32_e32 v4, v0
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB12_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB12_4
; GCN3-NEXT: .LBB12_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB12_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB12_2
; GCN3-NEXT: .LBB12_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_nop 0
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw xchg ptr %ptr, double %in seq_cst
ret double %result
}
define double @flat_atomic_xchg_f64_ret_offset(ptr %out, double %in) {
; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB13_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB13_4
; GCN1-NEXT: .LBB13_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB13_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB13_2
; GCN1-NEXT: .LBB13_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB13_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB13_4
; GCN2-NEXT: .LBB13_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB13_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB13_2
; GCN2-NEXT: .LBB13_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB13_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB13_4
; GCN3-NEXT: .LBB13_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB13_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB13_2
; GCN3-NEXT: .LBB13_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_nop 0
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr %out, i32 4
%result = atomicrmw xchg ptr %gep, double %in seq_cst
ret double %result
}
define amdgpu_gfx void @flat_atomic_xchg_f64_noret_scalar(ptr inreg %ptr, double inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_f64_noret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_mov_b64 s[34:35], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB14_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB14_4
; GCN1-NEXT: .LBB14_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB14_3: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execnz .LBB14_2
; GCN1-NEXT: .LBB14_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: v_mov_b32_e32 v0, s6
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v0, s7
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f64_noret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_mov_b64 s[34:35], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB14_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB14_4
; GCN2-NEXT: .LBB14_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB14_3: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execnz .LBB14_2
; GCN2-NEXT: .LBB14_4: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v0, s7
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f64_noret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_mov_b64 s[34:35], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB14_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB14_4
; GCN3-NEXT: .LBB14_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB14_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execnz .LBB14_2
; GCN3-NEXT: .LBB14_4: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s7
; GCN3-NEXT: v_mov_b32_e32 v1, s34
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw xchg ptr %ptr, double %in seq_cst
ret void
}
define amdgpu_gfx void @flat_atomic_xchg_f64_noret_offset_scalar(ptr inreg %out, double inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_mov_b64 s[36:37], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB15_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB15_4
; GCN1-NEXT: .LBB15_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB15_3: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execnz .LBB15_2
; GCN1-NEXT: .LBB15_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v0, s6
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v0, s7
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_mov_b64 s[36:37], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB15_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB15_4
; GCN2-NEXT: .LBB15_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB15_3: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execnz .LBB15_2
; GCN2-NEXT: .LBB15_4: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v0, s7
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_mov_b64 s[36:37], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB15_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB15_4
; GCN3-NEXT: .LBB15_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB15_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s35
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execnz .LBB15_2
; GCN3-NEXT: .LBB15_4: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s7
; GCN3-NEXT: v_mov_b32_e32 v1, s34
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr %out, i32 4
%tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst
ret void
}
define amdgpu_gfx double @flat_atomic_xchg_f64_ret_scalar(ptr inreg %ptr, double inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_f64_ret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB16_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execz .LBB16_3
; GCN1-NEXT: s_branch .LBB16_4
; GCN1-NEXT: .LBB16_2:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: .LBB16_3: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s6
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v2, s7
; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB16_4: ; %atomicrmw.end
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f64_ret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB16_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execz .LBB16_3
; GCN2-NEXT: s_branch .LBB16_4
; GCN2-NEXT: .LBB16_2:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: .LBB16_3: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v4, s6
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v2, s7
; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB16_4: ; %atomicrmw.end
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f64_ret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB16_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execz .LBB16_3
; GCN3-NEXT: s_branch .LBB16_4
; GCN3-NEXT: .LBB16_2:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: .LBB16_3: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v3, s6
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB16_4: ; %atomicrmw.end
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw xchg ptr %ptr, double %in seq_cst
ret double %result
}
define amdgpu_gfx double @flat_atomic_xchg_f64_ret_offset_scalar(ptr inreg %out, double inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB17_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execz .LBB17_3
; GCN1-NEXT: s_branch .LBB17_4
; GCN1-NEXT: .LBB17_2:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: .LBB17_3: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s6
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v2, s7
; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB17_4: ; %atomicrmw.end
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB17_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execz .LBB17_3
; GCN2-NEXT: s_branch .LBB17_4
; GCN2-NEXT: .LBB17_2:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: .LBB17_3: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v4, s6
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v2, s7
; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB17_4: ; %atomicrmw.end
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB17_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s35
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execz .LBB17_3
; GCN3-NEXT: s_branch .LBB17_4
; GCN3-NEXT: .LBB17_2:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: .LBB17_3: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v3, s6
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB17_4: ; %atomicrmw.end
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr %out, i32 4
%result = atomicrmw xchg ptr %gep, double %in seq_cst
ret double %result
}
define void @flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory(ptr %out, double %in) {
; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB18_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB18_4
; GCN1-NEXT: .LBB18_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB18_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB18_2
; GCN1-NEXT: .LBB18_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0
; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB18_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB18_4
; GCN2-NEXT: .LBB18_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB18_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB18_2
; GCN2-NEXT: .LBB18_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0
; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB18_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB18_4
; GCN3-NEXT: .LBB18_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB18_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB18_2
; GCN3-NEXT: .LBB18_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr %out, i64 4
%tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst, !amdgpu.no.remote.memory !0
ret void
}
define double @flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory(ptr %out, double %in) {
; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB19_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB19_4
; GCN1-NEXT: .LBB19_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB19_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB19_2
; GCN1-NEXT: .LBB19_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB19_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB19_4
; GCN2-NEXT: .LBB19_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB19_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB19_2
; GCN2-NEXT: .LBB19_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB19_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB19_4
; GCN3-NEXT: .LBB19_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB19_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB19_2
; GCN3-NEXT: .LBB19_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_nop 0
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr double, ptr %out, i64 4
%result = atomicrmw xchg ptr %gep, double %in seq_cst, !amdgpu.no.remote.memory !0
ret double %result
}
; ---------------------------------------------------------------------
; atomicrmw add
; ---------------------------------------------------------------------
define void @flat_atomic_add_i64_noret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_add_i64_noret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB20_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB20_4
; GCN1-NEXT: .LBB20_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB20_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB20_2
; GCN1-NEXT: .LBB20_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i64_noret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB20_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB20_4
; GCN2-NEXT: .LBB20_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB20_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB20_2
; GCN2-NEXT: .LBB20_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i64_noret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB20_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB20_4
; GCN3-NEXT: .LBB20_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB20_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB20_2
; GCN3-NEXT: .LBB20_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw add ptr %ptr, i64 %in seq_cst
ret void
}
define void @flat_atomic_add_i64_noret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_add_i64_noret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB21_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB21_4
; GCN1-NEXT: .LBB21_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB21_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB21_2
; GCN1-NEXT: .LBB21_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i64_noret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB21_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB21_4
; GCN2-NEXT: .LBB21_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB21_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB21_2
; GCN2-NEXT: .LBB21_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i64_noret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB21_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB21_4
; GCN3-NEXT: .LBB21_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB21_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB21_2
; GCN3-NEXT: .LBB21_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst
ret void
}
define i64 @flat_atomic_add_i64_ret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_add_i64_ret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_mov_b32_e32 v5, v1
; GCN1-NEXT: v_mov_b32_e32 v4, v0
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB22_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB22_4
; GCN1-NEXT: .LBB22_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB22_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB22_2
; GCN1-NEXT: .LBB22_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i64_ret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_mov_b32_e32 v5, v1
; GCN2-NEXT: v_mov_b32_e32 v4, v0
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB22_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB22_4
; GCN2-NEXT: .LBB22_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB22_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB22_2
; GCN2-NEXT: .LBB22_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v5, v1
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_mov_b32_e32 v4, v0
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB22_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB22_4
; GCN3-NEXT: .LBB22_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB22_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB22_2
; GCN3-NEXT: .LBB22_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw add ptr %ptr, i64 %in seq_cst
ret i64 %result
}
define i64 @flat_atomic_add_i64_ret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_add_i64_ret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB23_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB23_4
; GCN1-NEXT: .LBB23_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB23_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB23_2
; GCN1-NEXT: .LBB23_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i64_ret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB23_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB23_4
; GCN2-NEXT: .LBB23_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB23_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB23_2
; GCN2-NEXT: .LBB23_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i64_ret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB23_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB23_4
; GCN3-NEXT: .LBB23_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB23_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB23_2
; GCN3-NEXT: .LBB23_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw add ptr %gep, i64 %in seq_cst
ret i64 %result
}
define amdgpu_gfx void @flat_atomic_add_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_add_i64_noret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_mov_b64 s[34:35], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB24_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB24_4
; GCN1-NEXT: .LBB24_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB24_3: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execnz .LBB24_2
; GCN1-NEXT: .LBB24_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_add_i32_e32 v1, vcc, s6, v1
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i64_noret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_mov_b64 s[34:35], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB24_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB24_4
; GCN2-NEXT: .LBB24_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB24_3: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execnz .LBB24_2
; GCN2-NEXT: .LBB24_4: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_add_u32_e32 v1, vcc, s6, v1
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i64_noret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_mov_b64 s[34:35], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB24_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB24_4
; GCN3-NEXT: .LBB24_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB24_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execnz .LBB24_2
; GCN3-NEXT: .LBB24_4: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, s6, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v3, vcc
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw add ptr %ptr, i64 %in seq_cst
ret void
}
define amdgpu_gfx void @flat_atomic_add_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_add_i64_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_mov_b64 s[36:37], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB25_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB25_4
; GCN1-NEXT: .LBB25_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB25_3: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execnz .LBB25_2
; GCN1-NEXT: .LBB25_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_add_i32_e32 v1, vcc, s6, v1
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i64_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_mov_b64 s[36:37], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB25_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB25_4
; GCN2-NEXT: .LBB25_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB25_3: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execnz .LBB25_2
; GCN2-NEXT: .LBB25_4: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_add_u32_e32 v1, vcc, s6, v1
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i64_noret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_mov_b64 s[36:37], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB25_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB25_4
; GCN3-NEXT: .LBB25_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB25_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s35
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execnz .LBB25_2
; GCN3-NEXT: .LBB25_4: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, s6, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v3, vcc
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst
ret void
}
define amdgpu_gfx i64 @flat_atomic_add_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_add_i64_ret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB26_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execz .LBB26_3
; GCN1-NEXT: s_branch .LBB26_4
; GCN1-NEXT: .LBB26_2:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: .LBB26_3: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_add_i32_e32 v5, vcc, s6, v0
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB26_4: ; %atomicrmw.end
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i64_ret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB26_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execz .LBB26_3
; GCN2-NEXT: s_branch .LBB26_4
; GCN2-NEXT: .LBB26_2:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: .LBB26_3: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_add_u32_e32 v5, vcc, s6, v0
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB26_4: ; %atomicrmw.end
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i64_ret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB26_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execz .LBB26_3
; GCN3-NEXT: s_branch .LBB26_4
; GCN3-NEXT: .LBB26_2:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: .LBB26_3: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, s6, v0
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB26_4: ; %atomicrmw.end
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw add ptr %ptr, i64 %in seq_cst
ret i64 %result
}
define amdgpu_gfx i64 @flat_atomic_add_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_add_i64_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB27_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execz .LBB27_3
; GCN1-NEXT: s_branch .LBB27_4
; GCN1-NEXT: .LBB27_2:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: .LBB27_3: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_add_i32_e32 v5, vcc, s6, v0
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB27_4: ; %atomicrmw.end
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i64_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB27_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execz .LBB27_3
; GCN2-NEXT: s_branch .LBB27_4
; GCN2-NEXT: .LBB27_2:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: .LBB27_3: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_add_u32_e32 v5, vcc, s6, v0
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB27_4: ; %atomicrmw.end
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i64_ret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB27_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s35
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execz .LBB27_3
; GCN3-NEXT: s_branch .LBB27_4
; GCN3-NEXT: .LBB27_2:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: .LBB27_3: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, s6, v0
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB27_4: ; %atomicrmw.end
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw add ptr %gep, i64 %in seq_cst
ret i64 %result
}
define void @flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB28_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB28_4
; GCN1-NEXT: .LBB28_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB28_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB28_2
; GCN1-NEXT: .LBB28_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB28_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB28_4
; GCN2-NEXT: .LBB28_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB28_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB28_2
; GCN2-NEXT: .LBB28_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB28_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB28_4
; GCN3-NEXT: .LBB28_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB28_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB28_2
; GCN3-NEXT: .LBB28_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
ret void
}
define i64 @flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB29_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB29_4
; GCN1-NEXT: .LBB29_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB29_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB29_2
; GCN1-NEXT: .LBB29_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB29_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB29_4
; GCN2-NEXT: .LBB29_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB29_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB29_2
; GCN2-NEXT: .LBB29_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB29_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB29_4
; GCN3-NEXT: .LBB29_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB29_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB29_2
; GCN3-NEXT: .LBB29_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw add ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
ret i64 %result
}
; ---------------------------------------------------------------------
; atomicrmw sub
; ---------------------------------------------------------------------
define void @flat_atomic_sub_i64_noret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_sub_i64_noret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB30_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB30_4
; GCN1-NEXT: .LBB30_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB30_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB30_2
; GCN1-NEXT: .LBB30_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i64_noret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB30_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB30_4
; GCN2-NEXT: .LBB30_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB30_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB30_2
; GCN2-NEXT: .LBB30_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i64_noret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB30_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB30_4
; GCN3-NEXT: .LBB30_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB30_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB30_2
; GCN3-NEXT: .LBB30_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst
ret void
}
define void @flat_atomic_sub_i64_noret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_sub_i64_noret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB31_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB31_4
; GCN1-NEXT: .LBB31_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB31_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB31_2
; GCN1-NEXT: .LBB31_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i64_noret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB31_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB31_4
; GCN2-NEXT: .LBB31_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB31_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB31_2
; GCN2-NEXT: .LBB31_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i64_noret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB31_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB31_4
; GCN3-NEXT: .LBB31_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB31_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB31_2
; GCN3-NEXT: .LBB31_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst
ret void
}
define i64 @flat_atomic_sub_i64_ret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_sub_i64_ret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_mov_b32_e32 v5, v1
; GCN1-NEXT: v_mov_b32_e32 v4, v0
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB32_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB32_4
; GCN1-NEXT: .LBB32_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB32_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB32_2
; GCN1-NEXT: .LBB32_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i64_ret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_mov_b32_e32 v5, v1
; GCN2-NEXT: v_mov_b32_e32 v4, v0
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB32_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB32_4
; GCN2-NEXT: .LBB32_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB32_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB32_2
; GCN2-NEXT: .LBB32_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v5, v1
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_mov_b32_e32 v4, v0
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB32_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB32_4
; GCN3-NEXT: .LBB32_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB32_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB32_2
; GCN3-NEXT: .LBB32_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw sub ptr %ptr, i64 %in seq_cst
ret i64 %result
}
define i64 @flat_atomic_sub_i64_ret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_sub_i64_ret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB33_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB33_4
; GCN1-NEXT: .LBB33_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB33_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB33_2
; GCN1-NEXT: .LBB33_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i64_ret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB33_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB33_4
; GCN2-NEXT: .LBB33_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB33_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB33_2
; GCN2-NEXT: .LBB33_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i64_ret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB33_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB33_4
; GCN3-NEXT: .LBB33_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB33_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB33_2
; GCN3-NEXT: .LBB33_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw sub ptr %gep, i64 %in seq_cst
ret i64 %result
}
define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_sub_i64_noret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_mov_b64 s[34:35], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB34_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB34_4
; GCN1-NEXT: .LBB34_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB34_3: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execnz .LBB34_2
; GCN1-NEXT: .LBB34_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_subrev_i32_e32 v1, vcc, s6, v1
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i64_noret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_mov_b64 s[34:35], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB34_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB34_4
; GCN2-NEXT: .LBB34_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB34_3: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execnz .LBB34_2
; GCN2-NEXT: .LBB34_4: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_subrev_u32_e32 v1, vcc, s6, v1
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i64_noret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_mov_b64 s[34:35], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB34_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB34_4
; GCN3-NEXT: .LBB34_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB34_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execnz .LBB34_2
; GCN3-NEXT: .LBB34_4: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_subrev_co_u32_e32 v1, vcc, s6, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v3, vcc
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst
ret void
}
define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_mov_b64 s[36:37], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB35_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB35_4
; GCN1-NEXT: .LBB35_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB35_3: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execnz .LBB35_2
; GCN1-NEXT: .LBB35_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_subrev_i32_e32 v1, vcc, s6, v1
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_mov_b64 s[36:37], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB35_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB35_4
; GCN2-NEXT: .LBB35_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB35_3: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execnz .LBB35_2
; GCN2-NEXT: .LBB35_4: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_subrev_u32_e32 v1, vcc, s6, v1
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_mov_b64 s[36:37], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB35_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB35_4
; GCN3-NEXT: .LBB35_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB35_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s35
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execnz .LBB35_2
; GCN3-NEXT: .LBB35_4: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_subrev_co_u32_e32 v1, vcc, s6, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v3, vcc
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst
ret void
}
define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_sub_i64_ret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB36_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execz .LBB36_3
; GCN1-NEXT: s_branch .LBB36_4
; GCN1-NEXT: .LBB36_2:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: .LBB36_3: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v0
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB36_4: ; %atomicrmw.end
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i64_ret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB36_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execz .LBB36_3
; GCN2-NEXT: s_branch .LBB36_4
; GCN2-NEXT: .LBB36_2:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: .LBB36_3: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v0
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB36_4: ; %atomicrmw.end
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i64_ret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB36_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execz .LBB36_3
; GCN3-NEXT: s_branch .LBB36_4
; GCN3-NEXT: .LBB36_2:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: .LBB36_3: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v0
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB36_4: ; %atomicrmw.end
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw sub ptr %ptr, i64 %in seq_cst
ret i64 %result
}
define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB37_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execz .LBB37_3
; GCN1-NEXT: s_branch .LBB37_4
; GCN1-NEXT: .LBB37_2:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: .LBB37_3: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v0
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB37_4: ; %atomicrmw.end
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB37_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execz .LBB37_3
; GCN2-NEXT: s_branch .LBB37_4
; GCN2-NEXT: .LBB37_2:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: .LBB37_3: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v0
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc
; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB37_4: ; %atomicrmw.end
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB37_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s35
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execz .LBB37_3
; GCN3-NEXT: s_branch .LBB37_4
; GCN3-NEXT: .LBB37_2:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: .LBB37_3: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v0
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB37_4: ; %atomicrmw.end
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw sub ptr %gep, i64 %in seq_cst
ret i64 %result
}
define void @flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB38_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB38_4
; GCN1-NEXT: .LBB38_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB38_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB38_2
; GCN1-NEXT: .LBB38_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB38_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB38_4
; GCN2-NEXT: .LBB38_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB38_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB38_2
; GCN2-NEXT: .LBB38_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc
; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB38_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB38_4
; GCN3-NEXT: .LBB38_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB38_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB38_2
; GCN3-NEXT: .LBB38_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
ret void
}
define i64 @flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB39_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB39_4
; GCN1-NEXT: .LBB39_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB39_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB39_2
; GCN1-NEXT: .LBB39_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB39_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB39_4
; GCN2-NEXT: .LBB39_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB39_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB39_2
; GCN2-NEXT: .LBB39_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB39_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB39_4
; GCN3-NEXT: .LBB39_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB39_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB39_2
; GCN3-NEXT: .LBB39_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw sub ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
ret i64 %result
}
; ---------------------------------------------------------------------
; atomicrmw and
; ---------------------------------------------------------------------
define void @flat_atomic_and_i64_noret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_and_i64_noret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB40_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB40_4
; GCN1-NEXT: .LBB40_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB40_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB40_2
; GCN1-NEXT: .LBB40_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i64_noret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB40_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB40_4
; GCN2-NEXT: .LBB40_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB40_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB40_2
; GCN2-NEXT: .LBB40_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i64_noret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB40_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB40_4
; GCN3-NEXT: .LBB40_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB40_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB40_2
; GCN3-NEXT: .LBB40_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst
ret void
}
define void @flat_atomic_and_i64_noret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_and_i64_noret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB41_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB41_4
; GCN1-NEXT: .LBB41_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB41_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB41_2
; GCN1-NEXT: .LBB41_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i64_noret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB41_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB41_4
; GCN2-NEXT: .LBB41_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB41_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB41_2
; GCN2-NEXT: .LBB41_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i64_noret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB41_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB41_4
; GCN3-NEXT: .LBB41_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB41_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB41_2
; GCN3-NEXT: .LBB41_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst
ret void
}
define i64 @flat_atomic_and_i64_ret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_and_i64_ret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_mov_b32_e32 v5, v1
; GCN1-NEXT: v_mov_b32_e32 v4, v0
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB42_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB42_4
; GCN1-NEXT: .LBB42_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB42_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB42_2
; GCN1-NEXT: .LBB42_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, v1, v3
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i64_ret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_mov_b32_e32 v5, v1
; GCN2-NEXT: v_mov_b32_e32 v4, v0
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB42_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB42_4
; GCN2-NEXT: .LBB42_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB42_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB42_2
; GCN2-NEXT: .LBB42_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, v1, v3
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v5, v1
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_mov_b32_e32 v4, v0
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB42_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB42_4
; GCN3-NEXT: .LBB42_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB42_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB42_2
; GCN3-NEXT: .LBB42_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw and ptr %ptr, i64 %in seq_cst
ret i64 %result
}
define i64 @flat_atomic_and_i64_ret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_and_i64_ret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB43_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB43_4
; GCN1-NEXT: .LBB43_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB43_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB43_2
; GCN1-NEXT: .LBB43_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, v1, v3
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i64_ret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB43_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB43_4
; GCN2-NEXT: .LBB43_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB43_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB43_2
; GCN2-NEXT: .LBB43_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, v1, v3
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i64_ret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB43_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB43_4
; GCN3-NEXT: .LBB43_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB43_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB43_2
; GCN3-NEXT: .LBB43_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw and ptr %gep, i64 %in seq_cst
ret i64 %result
}
define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_and_i64_noret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_mov_b64 s[34:35], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB44_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB44_4
; GCN1-NEXT: .LBB44_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB44_3: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execnz .LBB44_2
; GCN1-NEXT: .LBB44_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, s6, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, s7, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i64_noret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_mov_b64 s[34:35], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB44_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB44_4
; GCN2-NEXT: .LBB44_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB44_3: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execnz .LBB44_2
; GCN2-NEXT: .LBB44_4: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, s6, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, s7, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i64_noret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_mov_b64 s[34:35], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB44_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB44_4
; GCN3-NEXT: .LBB44_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB44_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execnz .LBB44_2
; GCN3-NEXT: .LBB44_4: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v1, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, s6, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst
ret void
}
define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_and_i64_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_mov_b64 s[36:37], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB45_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB45_4
; GCN1-NEXT: .LBB45_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB45_3: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execnz .LBB45_2
; GCN1-NEXT: .LBB45_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, s6, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, s7, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i64_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_mov_b64 s[36:37], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB45_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB45_4
; GCN2-NEXT: .LBB45_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB45_3: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execnz .LBB45_2
; GCN2-NEXT: .LBB45_4: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, s6, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, s7, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i64_noret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_mov_b64 s[36:37], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB45_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB45_4
; GCN3-NEXT: .LBB45_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB45_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s35
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execnz .LBB45_2
; GCN3-NEXT: .LBB45_4: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v1, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, s6, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst
ret void
}
define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_and_i64_ret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB46_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execz .LBB46_3
; GCN1-NEXT: s_branch .LBB46_4
; GCN1-NEXT: .LBB46_2:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: .LBB46_3: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v4, s6, v0
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v5, s7, v1
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB46_4: ; %atomicrmw.end
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i64_ret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB46_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execz .LBB46_3
; GCN2-NEXT: s_branch .LBB46_4
; GCN2-NEXT: .LBB46_2:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: .LBB46_3: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v4, s6, v0
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v5, s7, v1
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB46_4: ; %atomicrmw.end
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i64_ret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB46_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execz .LBB46_3
; GCN3-NEXT: s_branch .LBB46_4
; GCN3-NEXT: .LBB46_2:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: .LBB46_3: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v3, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v4, s6, v0
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB46_4: ; %atomicrmw.end
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw and ptr %ptr, i64 %in seq_cst
ret i64 %result
}
define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_and_i64_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB47_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execz .LBB47_3
; GCN1-NEXT: s_branch .LBB47_4
; GCN1-NEXT: .LBB47_2:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: .LBB47_3: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v4, s6, v0
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v5, s7, v1
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB47_4: ; %atomicrmw.end
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i64_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB47_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execz .LBB47_3
; GCN2-NEXT: s_branch .LBB47_4
; GCN2-NEXT: .LBB47_2:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: .LBB47_3: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v4, s6, v0
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v5, s7, v1
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB47_4: ; %atomicrmw.end
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i64_ret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB47_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s35
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execz .LBB47_3
; GCN3-NEXT: s_branch .LBB47_4
; GCN3-NEXT: .LBB47_2:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: .LBB47_3: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v3, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v4, s6, v0
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB47_4: ; %atomicrmw.end
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw and ptr %gep, i64 %in seq_cst
ret i64 %result
}
define void @flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB48_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB48_4
; GCN1-NEXT: .LBB48_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB48_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB48_2
; GCN1-NEXT: .LBB48_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB48_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB48_4
; GCN2-NEXT: .LBB48_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB48_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB48_2
; GCN2-NEXT: .LBB48_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB48_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB48_4
; GCN3-NEXT: .LBB48_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB48_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB48_2
; GCN3-NEXT: .LBB48_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
ret void
}
define i64 @flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB49_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB49_4
; GCN1-NEXT: .LBB49_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB49_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB49_2
; GCN1-NEXT: .LBB49_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, v1, v3
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB49_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB49_4
; GCN2-NEXT: .LBB49_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB49_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB49_2
; GCN2-NEXT: .LBB49_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, v1, v3
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB49_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB49_4
; GCN3-NEXT: .LBB49_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB49_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB49_2
; GCN3-NEXT: .LBB49_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw and ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
ret i64 %result
}
; ---------------------------------------------------------------------
; atomicrmw nand
; ---------------------------------------------------------------------
define void @flat_atomic_nand_i64_noret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_nand_i64_noret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB50_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB50_6
; GCN1-NEXT: .LBB50_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB50_3: ; %atomicrmw.global
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v7, v[4:5]
; GCN1-NEXT: flat_load_dword v6, v[0:1]
; GCN1-NEXT: s_mov_b64 s[6:7], 0
; GCN1-NEXT: .LBB50_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v4, v7, v3
; GCN1-NEXT: v_and_b32_e32 v8, v6, v2
; GCN1-NEXT: v_not_b32_e32 v5, v4
; GCN1-NEXT: v_not_b32_e32 v4, v8
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v5
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN1-NEXT: v_mov_b32_e32 v6, v4
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN1-NEXT: s_cbranch_execnz .LBB50_4
; GCN1-NEXT: ; %bb.5: ; %Flow
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB50_2
; GCN1-NEXT: .LBB50_6: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
; GCN1-NEXT: v_not_b32_e32 v2, v2
; GCN1-NEXT: v_not_b32_e32 v3, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_noret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB50_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB50_6
; GCN2-NEXT: .LBB50_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB50_3: ; %atomicrmw.global
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v7, v[4:5]
; GCN2-NEXT: flat_load_dword v6, v[0:1]
; GCN2-NEXT: s_mov_b64 s[6:7], 0
; GCN2-NEXT: .LBB50_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v4, v7, v3
; GCN2-NEXT: v_and_b32_e32 v8, v6, v2
; GCN2-NEXT: v_not_b32_e32 v5, v4
; GCN2-NEXT: v_not_b32_e32 v4, v8
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v5
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN2-NEXT: v_mov_b32_e32 v6, v4
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN2-NEXT: s_cbranch_execnz .LBB50_4
; GCN2-NEXT: ; %bb.5: ; %Flow
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB50_2
; GCN2-NEXT: .LBB50_6: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
; GCN2-NEXT: v_not_b32_e32 v2, v2
; GCN2-NEXT: v_not_b32_e32 v3, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_noret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB50_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB50_6
; GCN3-NEXT: .LBB50_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB50_3: ; %atomicrmw.global
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
; GCN3-NEXT: s_mov_b64 s[6:7], 0
; GCN3-NEXT: .LBB50_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v4, v7, v3
; GCN3-NEXT: v_and_b32_e32 v8, v6, v2
; GCN3-NEXT: v_not_b32_e32 v5, v4
; GCN3-NEXT: v_not_b32_e32 v4, v8
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN3-NEXT: s_cbranch_execnz .LBB50_4
; GCN3-NEXT: ; %bb.5: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB50_2
; GCN3-NEXT: .LBB50_6: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
; GCN3-NEXT: v_not_b32_e32 v2, v2
; GCN3-NEXT: v_not_b32_e32 v1, v1
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw nand ptr %ptr, i64 %in seq_cst
ret void
}
define void @flat_atomic_nand_i64_noret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_nand_i64_noret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB51_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB51_6
; GCN1-NEXT: .LBB51_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB51_3: ; %atomicrmw.global
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v7, v[4:5]
; GCN1-NEXT: flat_load_dword v6, v[0:1]
; GCN1-NEXT: s_mov_b64 s[6:7], 0
; GCN1-NEXT: .LBB51_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v4, v7, v3
; GCN1-NEXT: v_and_b32_e32 v8, v6, v2
; GCN1-NEXT: v_not_b32_e32 v5, v4
; GCN1-NEXT: v_not_b32_e32 v4, v8
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v5
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN1-NEXT: v_mov_b32_e32 v6, v4
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN1-NEXT: s_cbranch_execnz .LBB51_4
; GCN1-NEXT: ; %bb.5: ; %Flow
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB51_2
; GCN1-NEXT: .LBB51_6: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
; GCN1-NEXT: v_not_b32_e32 v2, v2
; GCN1-NEXT: v_not_b32_e32 v3, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_noret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB51_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB51_6
; GCN2-NEXT: .LBB51_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB51_3: ; %atomicrmw.global
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v7, v[4:5]
; GCN2-NEXT: flat_load_dword v6, v[0:1]
; GCN2-NEXT: s_mov_b64 s[6:7], 0
; GCN2-NEXT: .LBB51_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v4, v7, v3
; GCN2-NEXT: v_and_b32_e32 v8, v6, v2
; GCN2-NEXT: v_not_b32_e32 v5, v4
; GCN2-NEXT: v_not_b32_e32 v4, v8
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v5
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN2-NEXT: v_mov_b32_e32 v6, v4
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN2-NEXT: s_cbranch_execnz .LBB51_4
; GCN2-NEXT: ; %bb.5: ; %Flow
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB51_2
; GCN2-NEXT: .LBB51_6: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
; GCN2-NEXT: v_not_b32_e32 v2, v2
; GCN2-NEXT: v_not_b32_e32 v3, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_noret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB51_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB51_6
; GCN3-NEXT: .LBB51_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB51_3: ; %atomicrmw.global
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
; GCN3-NEXT: s_mov_b64 s[6:7], 0
; GCN3-NEXT: .LBB51_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v4, v7, v3
; GCN3-NEXT: v_and_b32_e32 v8, v6, v2
; GCN3-NEXT: v_not_b32_e32 v5, v4
; GCN3-NEXT: v_not_b32_e32 v4, v8
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN3-NEXT: s_cbranch_execnz .LBB51_4
; GCN3-NEXT: ; %bb.5: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB51_2
; GCN3-NEXT: .LBB51_6: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
; GCN3-NEXT: v_not_b32_e32 v2, v2
; GCN3-NEXT: v_not_b32_e32 v1, v1
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst
ret void
}
define void @flat_atomic_nand_i64_noret_offset__noalias_private(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_nand_i64_noret_offset__noalias_private:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v7, v[0:1]
; GCN1-NEXT: flat_load_dword v6, v[8:9]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB52_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v0, v7, v3
; GCN1-NEXT: v_and_b32_e32 v1, v6, v2
; GCN1-NEXT: v_not_b32_e32 v5, v0
; GCN1-NEXT: v_not_b32_e32 v4, v1
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v1
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v6, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB52_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_noret_offset__noalias_private:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v7, v[0:1]
; GCN2-NEXT: flat_load_dword v6, v[8:9]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB52_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v0, v7, v3
; GCN2-NEXT: v_and_b32_e32 v1, v6, v2
; GCN2-NEXT: v_not_b32_e32 v5, v0
; GCN2-NEXT: v_not_b32_e32 v4, v1
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v1
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v6, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB52_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_noret_offset__noalias_private:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB52_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v4, v7, v3
; GCN3-NEXT: v_and_b32_e32 v8, v6, v2
; GCN3-NEXT: v_not_b32_e32 v5, v4
; GCN3-NEXT: v_not_b32_e32 v4, v8
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB52_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define i64 @flat_atomic_nand_i64_ret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_nand_i64_ret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB53_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v5, v[4:5]
; GCN1-NEXT: flat_load_dword v4, v[0:1]
; GCN1-NEXT: s_mov_b64 s[6:7], 0
; GCN1-NEXT: .LBB53_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v7, v5
; GCN1-NEXT: v_mov_b32_e32 v6, v4
; GCN1-NEXT: v_and_b32_e32 v4, v7, v3
; GCN1-NEXT: v_and_b32_e32 v8, v6, v2
; GCN1-NEXT: v_not_b32_e32 v5, v4
; GCN1-NEXT: v_not_b32_e32 v4, v8
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN1-NEXT: s_cbranch_execnz .LBB53_2
; GCN1-NEXT: ; %bb.3: ; %Flow
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: .LBB53_4: ; %Flow3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB53_6
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
; GCN1-NEXT: v_not_b32_e32 v2, v2
; GCN1-NEXT: v_not_b32_e32 v3, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: .LBB53_6: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v0, v4
; GCN1-NEXT: v_mov_b32_e32 v1, v5
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_ret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB53_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v5, v[4:5]
; GCN2-NEXT: flat_load_dword v4, v[0:1]
; GCN2-NEXT: s_mov_b64 s[6:7], 0
; GCN2-NEXT: .LBB53_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v7, v5
; GCN2-NEXT: v_mov_b32_e32 v6, v4
; GCN2-NEXT: v_and_b32_e32 v4, v7, v3
; GCN2-NEXT: v_and_b32_e32 v8, v6, v2
; GCN2-NEXT: v_not_b32_e32 v5, v4
; GCN2-NEXT: v_not_b32_e32 v4, v8
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN2-NEXT: s_cbranch_execnz .LBB53_2
; GCN2-NEXT: ; %bb.3: ; %Flow
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: .LBB53_4: ; %Flow3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB53_6
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
; GCN2-NEXT: v_not_b32_e32 v2, v2
; GCN2-NEXT: v_not_b32_e32 v3, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: .LBB53_6: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v0, v4
; GCN2-NEXT: v_mov_b32_e32 v1, v5
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v5, v1
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_mov_b32_e32 v4, v0
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB53_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB53_6
; GCN3-NEXT: .LBB53_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB53_3: ; %atomicrmw.global
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
; GCN3-NEXT: s_mov_b64 s[6:7], 0
; GCN3-NEXT: .LBB53_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v1
; GCN3-NEXT: v_mov_b32_e32 v8, v0
; GCN3-NEXT: v_and_b32_e32 v0, v9, v3
; GCN3-NEXT: v_and_b32_e32 v1, v8, v2
; GCN3-NEXT: v_not_b32_e32 v7, v0
; GCN3-NEXT: v_not_b32_e32 v6, v1
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN3-NEXT: s_cbranch_execnz .LBB53_4
; GCN3-NEXT: ; %bb.5: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB53_2
; GCN3-NEXT: .LBB53_6: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
; GCN3-NEXT: v_not_b32_e32 v2, v2
; GCN3-NEXT: v_not_b32_e32 v3, v3
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw nand ptr %ptr, i64 %in seq_cst
ret i64 %result
}
define i64 @flat_atomic_nand_i64_ret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_nand_i64_ret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB54_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB54_6
; GCN1-NEXT: .LBB54_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB54_3: ; %atomicrmw.global
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[4:5]
; GCN1-NEXT: s_mov_b64 s[6:7], 0
; GCN1-NEXT: .LBB54_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_and_b32_e32 v0, v9, v3
; GCN1-NEXT: v_and_b32_e32 v1, v8, v2
; GCN1-NEXT: v_not_b32_e32 v7, v0
; GCN1-NEXT: v_not_b32_e32 v6, v1
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN1-NEXT: s_cbranch_execnz .LBB54_4
; GCN1-NEXT: ; %bb.5: ; %Flow
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB54_2
; GCN1-NEXT: .LBB54_6: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, v1, v3
; GCN1-NEXT: v_not_b32_e32 v2, v2
; GCN1-NEXT: v_not_b32_e32 v3, v3
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_ret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB54_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB54_6
; GCN2-NEXT: .LBB54_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB54_3: ; %atomicrmw.global
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[4:5]
; GCN2-NEXT: s_mov_b64 s[6:7], 0
; GCN2-NEXT: .LBB54_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_and_b32_e32 v0, v9, v3
; GCN2-NEXT: v_and_b32_e32 v1, v8, v2
; GCN2-NEXT: v_not_b32_e32 v7, v0
; GCN2-NEXT: v_not_b32_e32 v6, v1
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN2-NEXT: s_cbranch_execnz .LBB54_4
; GCN2-NEXT: ; %bb.5: ; %Flow
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB54_2
; GCN2-NEXT: .LBB54_6: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, v1, v3
; GCN2-NEXT: v_not_b32_e32 v2, v2
; GCN2-NEXT: v_not_b32_e32 v3, v3
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_ret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB54_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB54_6
; GCN3-NEXT: .LBB54_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB54_3: ; %atomicrmw.global
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
; GCN3-NEXT: s_mov_b64 s[6:7], 0
; GCN3-NEXT: .LBB54_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v1
; GCN3-NEXT: v_mov_b32_e32 v8, v0
; GCN3-NEXT: v_and_b32_e32 v0, v9, v3
; GCN3-NEXT: v_and_b32_e32 v1, v8, v2
; GCN3-NEXT: v_not_b32_e32 v7, v0
; GCN3-NEXT: v_not_b32_e32 v6, v1
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN3-NEXT: s_cbranch_execnz .LBB54_4
; GCN3-NEXT: ; %bb.5: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB54_2
; GCN3-NEXT: .LBB54_6: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
; GCN3-NEXT: v_not_b32_e32 v2, v2
; GCN3-NEXT: v_not_b32_e32 v3, v3
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw nand ptr %gep, i64 %in seq_cst
ret i64 %result
}
define i64 @flat_atomic_nand_i64_ret_offset__noalias_private(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_nand_i64_ret_offset__noalias_private:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB55_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_and_b32_e32 v0, v9, v3
; GCN1-NEXT: v_and_b32_e32 v1, v8, v2
; GCN1-NEXT: v_not_b32_e32 v7, v0
; GCN1-NEXT: v_not_b32_e32 v6, v1
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB55_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_ret_offset__noalias_private:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB55_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_and_b32_e32 v0, v9, v3
; GCN2-NEXT: v_and_b32_e32 v1, v8, v2
; GCN2-NEXT: v_not_b32_e32 v7, v0
; GCN2-NEXT: v_not_b32_e32 v6, v1
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB55_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_ret_offset__noalias_private:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB55_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: v_and_b32_e32 v4, v7, v3
; GCN3-NEXT: v_and_b32_e32 v8, v6, v2
; GCN3-NEXT: v_not_b32_e32 v5, v4
; GCN3-NEXT: v_not_b32_e32 v4, v8
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB55_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v0, v4
; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_nand_i64_noret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_mov_b64 s[34:35], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB56_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccnz .LBB56_6
; GCN1-NEXT: .LBB56_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB56_3: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v4, s4
; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB56_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
; GCN1-NEXT: v_not_b32_e32 v1, v0
; GCN1-NEXT: v_not_b32_e32 v0, v6
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB56_4
; GCN1-NEXT: ; %bb.5: ; %Flow
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_branch .LBB56_2
; GCN1-NEXT: .LBB56_6: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, s6, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, s7, v3
; GCN1-NEXT: v_not_b32_e32 v2, v2
; GCN1-NEXT: v_not_b32_e32 v3, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_noret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_mov_b64 s[34:35], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB56_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccnz .LBB56_6
; GCN2-NEXT: .LBB56_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB56_3: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v4, s4
; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB56_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
; GCN2-NEXT: v_not_b32_e32 v1, v0
; GCN2-NEXT: v_not_b32_e32 v0, v6
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB56_4
; GCN2-NEXT: ; %bb.5: ; %Flow
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_branch .LBB56_2
; GCN2-NEXT: .LBB56_6: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, s6, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, s7, v3
; GCN2-NEXT: v_not_b32_e32 v2, v2
; GCN2-NEXT: v_not_b32_e32 v3, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_noret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_mov_b64 s[34:35], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB56_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccnz .LBB56_6
; GCN3-NEXT: .LBB56_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB56_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB56_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
; GCN3-NEXT: v_not_b32_e32 v1, v0
; GCN3-NEXT: v_not_b32_e32 v0, v6
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB56_4
; GCN3-NEXT: ; %bb.5: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_branch .LBB56_2
; GCN3-NEXT: .LBB56_6: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v1, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, s6, v2
; GCN3-NEXT: v_not_b32_e32 v2, v2
; GCN3-NEXT: v_not_b32_e32 v1, v1
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw nand ptr %ptr, i64 %in seq_cst
ret void
}
define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_mov_b64 s[36:37], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB57_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccnz .LBB57_6
; GCN1-NEXT: .LBB57_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB57_3: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s36, s34, 4
; GCN1-NEXT: s_addc_u32 s37, s35, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s36
; GCN1-NEXT: v_mov_b32_e32 v1, s37
; GCN1-NEXT: v_mov_b32_e32 v4, s34
; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
; GCN1-NEXT: .LBB57_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
; GCN1-NEXT: v_not_b32_e32 v1, v0
; GCN1-NEXT: v_not_b32_e32 v0, v6
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB57_4
; GCN1-NEXT: ; %bb.5: ; %Flow
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_branch .LBB57_2
; GCN1-NEXT: .LBB57_6: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, s6, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, s7, v3
; GCN1-NEXT: v_not_b32_e32 v2, v2
; GCN1-NEXT: v_not_b32_e32 v3, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_mov_b64 s[36:37], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB57_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccnz .LBB57_6
; GCN2-NEXT: .LBB57_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB57_3: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s36, s34, 4
; GCN2-NEXT: s_addc_u32 s37, s35, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s36
; GCN2-NEXT: v_mov_b32_e32 v1, s37
; GCN2-NEXT: v_mov_b32_e32 v4, s34
; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
; GCN2-NEXT: .LBB57_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
; GCN2-NEXT: v_not_b32_e32 v1, v0
; GCN2-NEXT: v_not_b32_e32 v0, v6
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB57_4
; GCN2-NEXT: ; %bb.5: ; %Flow
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_branch .LBB57_2
; GCN2-NEXT: .LBB57_6: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, s6, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, s7, v3
; GCN2-NEXT: v_not_b32_e32 v2, v2
; GCN2-NEXT: v_not_b32_e32 v3, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_mov_b64 s[36:37], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB57_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccnz .LBB57_6
; GCN3-NEXT: .LBB57_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB57_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v4, s34
; GCN3-NEXT: v_mov_b32_e32 v5, s35
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
; GCN3-NEXT: .LBB57_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
; GCN3-NEXT: v_not_b32_e32 v1, v0
; GCN3-NEXT: v_not_b32_e32 v0, v6
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB57_4
; GCN3-NEXT: ; %bb.5: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_branch .LBB57_2
; GCN3-NEXT: .LBB57_6: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v1, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, s6, v2
; GCN3-NEXT: v_not_b32_e32 v2, v2
; GCN3-NEXT: v_not_b32_e32 v1, v1
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst
ret void
}
define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar__noalias_addrspace(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_nand_i64_noret_offset_scalar__noalias_addrspace:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_add_u32 s36, s4, 36
; GCN1-NEXT: s_addc_u32 s37, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s36
; GCN1-NEXT: v_mov_b32_e32 v1, s37
; GCN1-NEXT: v_mov_b32_e32 v4, s34
; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB58_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
; GCN1-NEXT: v_not_b32_e32 v1, v0
; GCN1-NEXT: v_not_b32_e32 v0, v6
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB58_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_noret_offset_scalar__noalias_addrspace:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_add_u32 s36, s4, 36
; GCN2-NEXT: s_addc_u32 s37, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s36
; GCN2-NEXT: v_mov_b32_e32 v1, s37
; GCN2-NEXT: v_mov_b32_e32 v4, s34
; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB58_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
; GCN2-NEXT: v_not_b32_e32 v1, v0
; GCN2-NEXT: v_not_b32_e32 v0, v6
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB58_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_noret_offset_scalar__noalias_addrspace:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB58_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
; GCN3-NEXT: v_not_b32_e32 v1, v0
; GCN3-NEXT: v_not_b32_e32 v0, v6
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB58_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_nand_i64_ret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB59_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB59_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v7, v1
; GCN1-NEXT: v_mov_b32_e32 v6, v0
; GCN1-NEXT: v_and_b32_e32 v0, s7, v7
; GCN1-NEXT: v_and_b32_e32 v1, s6, v6
; GCN1-NEXT: v_not_b32_e32 v5, v0
; GCN1-NEXT: v_not_b32_e32 v4, v1
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB59_2
; GCN1-NEXT: ; %bb.3: ; %Flow
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_branch .LBB59_6
; GCN1-NEXT: .LBB59_4:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_cbranch_execz .LBB59_6
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v4, s6, v0
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v5, s7, v1
; GCN1-NEXT: v_not_b32_e32 v4, v4
; GCN1-NEXT: v_not_b32_e32 v5, v5
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB59_6: ; %atomicrmw.phi
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_ret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB59_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB59_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v7, v1
; GCN2-NEXT: v_mov_b32_e32 v6, v0
; GCN2-NEXT: v_and_b32_e32 v0, s7, v7
; GCN2-NEXT: v_and_b32_e32 v1, s6, v6
; GCN2-NEXT: v_not_b32_e32 v5, v0
; GCN2-NEXT: v_not_b32_e32 v4, v1
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB59_2
; GCN2-NEXT: ; %bb.3: ; %Flow
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_branch .LBB59_6
; GCN2-NEXT: .LBB59_4:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_cbranch_execz .LBB59_6
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v4, s6, v0
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v5, s7, v1
; GCN2-NEXT: v_not_b32_e32 v4, v4
; GCN2-NEXT: v_not_b32_e32 v5, v5
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB59_6: ; %atomicrmw.phi
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_ret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB59_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB59_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v1
; GCN3-NEXT: v_mov_b32_e32 v6, v0
; GCN3-NEXT: v_and_b32_e32 v0, s7, v7
; GCN3-NEXT: v_and_b32_e32 v1, s6, v6
; GCN3-NEXT: v_not_b32_e32 v5, v0
; GCN3-NEXT: v_not_b32_e32 v4, v1
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB59_2
; GCN3-NEXT: ; %bb.3: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_branch .LBB59_6
; GCN3-NEXT: .LBB59_4:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_cbranch_execz .LBB59_6
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v3, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v4, s6, v0
; GCN3-NEXT: v_not_b32_e32 v4, v4
; GCN3-NEXT: v_not_b32_e32 v3, v3
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB59_6: ; %atomicrmw.phi
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw nand ptr %ptr, i64 %in seq_cst
ret i64 %result
}
define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB60_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s36, s34, 4
; GCN1-NEXT: s_addc_u32 s37, s35, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s36
; GCN1-NEXT: v_mov_b32_e32 v1, s37
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
; GCN1-NEXT: .LBB60_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v7, v1
; GCN1-NEXT: v_mov_b32_e32 v6, v0
; GCN1-NEXT: v_and_b32_e32 v0, s7, v7
; GCN1-NEXT: v_and_b32_e32 v1, s6, v6
; GCN1-NEXT: v_not_b32_e32 v5, v0
; GCN1-NEXT: v_not_b32_e32 v4, v1
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB60_2
; GCN1-NEXT: ; %bb.3: ; %Flow
; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_branch .LBB60_6
; GCN1-NEXT: .LBB60_4:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_cbranch_execz .LBB60_6
; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v4, s6, v0
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v5, s7, v1
; GCN1-NEXT: v_not_b32_e32 v4, v4
; GCN1-NEXT: v_not_b32_e32 v5, v5
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB60_6: ; %atomicrmw.phi
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB60_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s36, s34, 4
; GCN2-NEXT: s_addc_u32 s37, s35, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s36
; GCN2-NEXT: v_mov_b32_e32 v1, s37
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
; GCN2-NEXT: .LBB60_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v7, v1
; GCN2-NEXT: v_mov_b32_e32 v6, v0
; GCN2-NEXT: v_and_b32_e32 v0, s7, v7
; GCN2-NEXT: v_and_b32_e32 v1, s6, v6
; GCN2-NEXT: v_not_b32_e32 v5, v0
; GCN2-NEXT: v_not_b32_e32 v4, v1
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB60_2
; GCN2-NEXT: ; %bb.3: ; %Flow
; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_branch .LBB60_6
; GCN2-NEXT: .LBB60_4:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_cbranch_execz .LBB60_6
; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v4, s6, v0
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v5, s7, v1
; GCN2-NEXT: v_not_b32_e32 v4, v4
; GCN2-NEXT: v_not_b32_e32 v5, v5
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB60_6: ; %atomicrmw.phi
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB60_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: v_mov_b32_e32 v3, s35
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
; GCN3-NEXT: .LBB60_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v1
; GCN3-NEXT: v_mov_b32_e32 v6, v0
; GCN3-NEXT: v_and_b32_e32 v0, s7, v7
; GCN3-NEXT: v_and_b32_e32 v1, s6, v6
; GCN3-NEXT: v_not_b32_e32 v5, v0
; GCN3-NEXT: v_not_b32_e32 v4, v1
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB60_2
; GCN3-NEXT: ; %bb.3: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_branch .LBB60_6
; GCN3-NEXT: .LBB60_4:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_cbranch_execz .LBB60_6
; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v3, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v4, s6, v0
; GCN3-NEXT: v_not_b32_e32 v4, v4
; GCN3-NEXT: v_not_b32_e32 v3, v3
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB60_6: ; %atomicrmw.phi
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw nand ptr %gep, i64 %in seq_cst
ret i64 %result
}
define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar__noalias_private(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_nand_i64_ret_offset_scalar__noalias_private:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_add_u32 s36, s4, 36
; GCN1-NEXT: s_addc_u32 s37, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s36
; GCN1-NEXT: v_mov_b32_e32 v1, s37
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB61_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v7, v1
; GCN1-NEXT: v_mov_b32_e32 v6, v0
; GCN1-NEXT: v_and_b32_e32 v0, s7, v7
; GCN1-NEXT: v_and_b32_e32 v1, s6, v6
; GCN1-NEXT: v_not_b32_e32 v5, v0
; GCN1-NEXT: v_not_b32_e32 v4, v1
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB61_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_ret_offset_scalar__noalias_private:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_add_u32 s36, s4, 36
; GCN2-NEXT: s_addc_u32 s37, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s36
; GCN2-NEXT: v_mov_b32_e32 v1, s37
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB61_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v7, v1
; GCN2-NEXT: v_mov_b32_e32 v6, v0
; GCN2-NEXT: v_and_b32_e32 v0, s7, v7
; GCN2-NEXT: v_and_b32_e32 v1, s6, v6
; GCN2-NEXT: v_not_b32_e32 v5, v0
; GCN2-NEXT: v_not_b32_e32 v4, v1
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB61_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_ret_offset_scalar__noalias_private:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB61_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v1
; GCN3-NEXT: v_mov_b32_e32 v6, v0
; GCN3-NEXT: v_and_b32_e32 v0, s7, v7
; GCN3-NEXT: v_and_b32_e32 v1, s6, v6
; GCN3-NEXT: v_not_b32_e32 v5, v0
; GCN3-NEXT: v_not_b32_e32 v4, v1
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB61_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define void @flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB62_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB62_6
; GCN1-NEXT: .LBB62_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB62_3: ; %atomicrmw.global
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v7, v[4:5]
; GCN1-NEXT: flat_load_dword v6, v[0:1]
; GCN1-NEXT: s_mov_b64 s[6:7], 0
; GCN1-NEXT: .LBB62_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v4, v7, v3
; GCN1-NEXT: v_and_b32_e32 v8, v6, v2
; GCN1-NEXT: v_not_b32_e32 v5, v4
; GCN1-NEXT: v_not_b32_e32 v4, v8
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v5
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN1-NEXT: v_mov_b32_e32 v6, v4
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN1-NEXT: s_cbranch_execnz .LBB62_4
; GCN1-NEXT: ; %bb.5: ; %Flow
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB62_2
; GCN1-NEXT: .LBB62_6: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, v4, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, v5, v3
; GCN1-NEXT: v_not_b32_e32 v2, v2
; GCN1-NEXT: v_not_b32_e32 v3, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB62_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB62_6
; GCN2-NEXT: .LBB62_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB62_3: ; %atomicrmw.global
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v7, v[4:5]
; GCN2-NEXT: flat_load_dword v6, v[0:1]
; GCN2-NEXT: s_mov_b64 s[6:7], 0
; GCN2-NEXT: .LBB62_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v4, v7, v3
; GCN2-NEXT: v_and_b32_e32 v8, v6, v2
; GCN2-NEXT: v_not_b32_e32 v5, v4
; GCN2-NEXT: v_not_b32_e32 v4, v8
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v5
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN2-NEXT: v_mov_b32_e32 v6, v4
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN2-NEXT: s_cbranch_execnz .LBB62_4
; GCN2-NEXT: ; %bb.5: ; %Flow
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB62_2
; GCN2-NEXT: .LBB62_6: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, v4, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, v5, v3
; GCN2-NEXT: v_not_b32_e32 v2, v2
; GCN2-NEXT: v_not_b32_e32 v3, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB62_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB62_6
; GCN3-NEXT: .LBB62_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB62_3: ; %atomicrmw.global
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
; GCN3-NEXT: s_mov_b64 s[6:7], 0
; GCN3-NEXT: .LBB62_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v4, v7, v3
; GCN3-NEXT: v_and_b32_e32 v8, v6, v2
; GCN3-NEXT: v_not_b32_e32 v5, v4
; GCN3-NEXT: v_not_b32_e32 v4, v8
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN3-NEXT: s_cbranch_execnz .LBB62_4
; GCN3-NEXT: ; %bb.5: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB62_2
; GCN3-NEXT: .LBB62_6: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v1, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, v4, v2
; GCN3-NEXT: v_not_b32_e32 v2, v2
; GCN3-NEXT: v_not_b32_e32 v1, v1
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
ret void
}
define i64 @flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB63_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB63_6
; GCN1-NEXT: .LBB63_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB63_3: ; %atomicrmw.global
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[4:5]
; GCN1-NEXT: s_mov_b64 s[6:7], 0
; GCN1-NEXT: .LBB63_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_and_b32_e32 v0, v9, v3
; GCN1-NEXT: v_and_b32_e32 v1, v8, v2
; GCN1-NEXT: v_not_b32_e32 v7, v0
; GCN1-NEXT: v_not_b32_e32 v6, v1
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN1-NEXT: s_cbranch_execnz .LBB63_4
; GCN1-NEXT: ; %bb.5: ; %Flow
; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: ; implicit-def: $vgpr2
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB63_2
; GCN1-NEXT: .LBB63_6: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_and_b32_e32 v2, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v3, v1, v3
; GCN1-NEXT: v_not_b32_e32 v2, v2
; GCN1-NEXT: v_not_b32_e32 v3, v3
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB63_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB63_6
; GCN2-NEXT: .LBB63_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB63_3: ; %atomicrmw.global
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[4:5]
; GCN2-NEXT: s_mov_b64 s[6:7], 0
; GCN2-NEXT: .LBB63_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_and_b32_e32 v0, v9, v3
; GCN2-NEXT: v_and_b32_e32 v1, v8, v2
; GCN2-NEXT: v_not_b32_e32 v7, v0
; GCN2-NEXT: v_not_b32_e32 v6, v1
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN2-NEXT: s_cbranch_execnz .LBB63_4
; GCN2-NEXT: ; %bb.5: ; %Flow
; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: ; implicit-def: $vgpr2
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB63_2
; GCN2-NEXT: .LBB63_6: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_and_b32_e32 v2, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v3, v1, v3
; GCN2-NEXT: v_not_b32_e32 v2, v2
; GCN2-NEXT: v_not_b32_e32 v3, v3
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB63_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB63_6
; GCN3-NEXT: .LBB63_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB63_3: ; %atomicrmw.global
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
; GCN3-NEXT: s_mov_b64 s[6:7], 0
; GCN3-NEXT: .LBB63_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v1
; GCN3-NEXT: v_mov_b32_e32 v8, v0
; GCN3-NEXT: v_and_b32_e32 v0, v9, v3
; GCN3-NEXT: v_and_b32_e32 v1, v8, v2
; GCN3-NEXT: v_not_b32_e32 v7, v0
; GCN3-NEXT: v_not_b32_e32 v6, v1
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN3-NEXT: s_cbranch_execnz .LBB63_4
; GCN3-NEXT: ; %bb.5: ; %Flow
; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: ; implicit-def: $vgpr2
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB63_2
; GCN3-NEXT: .LBB63_6: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_and_b32_e32 v3, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v2, v0, v2
; GCN3-NEXT: v_not_b32_e32 v2, v2
; GCN3-NEXT: v_not_b32_e32 v3, v3
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw nand ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
ret i64 %result
}
; ---------------------------------------------------------------------
; atomicrmw or
; ---------------------------------------------------------------------
define void @flat_atomic_or_i64_noret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_or_i64_noret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB64_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB64_4
; GCN1-NEXT: .LBB64_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB64_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB64_2
; GCN1-NEXT: .LBB64_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_or_b32_e32 v2, v4, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_or_b32_e32 v3, v5, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i64_noret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB64_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB64_4
; GCN2-NEXT: .LBB64_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB64_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB64_2
; GCN2-NEXT: .LBB64_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_or_b32_e32 v2, v4, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_or_b32_e32 v3, v5, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i64_noret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB64_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB64_4
; GCN3-NEXT: .LBB64_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB64_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB64_2
; GCN3-NEXT: .LBB64_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_or_b32_e32 v1, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_or_b32_e32 v2, v4, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst
ret void
}
define void @flat_atomic_or_i64_noret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_or_i64_noret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB65_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB65_4
; GCN1-NEXT: .LBB65_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB65_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB65_2
; GCN1-NEXT: .LBB65_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_or_b32_e32 v2, v4, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_or_b32_e32 v3, v5, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i64_noret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB65_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB65_4
; GCN2-NEXT: .LBB65_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB65_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB65_2
; GCN2-NEXT: .LBB65_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_or_b32_e32 v2, v4, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_or_b32_e32 v3, v5, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i64_noret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB65_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB65_4
; GCN3-NEXT: .LBB65_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB65_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB65_2
; GCN3-NEXT: .LBB65_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_or_b32_e32 v1, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_or_b32_e32 v2, v4, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst
ret void
}
define i64 @flat_atomic_or_i64_ret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_or_i64_ret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_mov_b32_e32 v5, v1
; GCN1-NEXT: v_mov_b32_e32 v4, v0
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB66_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB66_4
; GCN1-NEXT: .LBB66_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB66_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB66_2
; GCN1-NEXT: .LBB66_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_or_b32_e32 v2, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_or_b32_e32 v3, v1, v3
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i64_ret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_mov_b32_e32 v5, v1
; GCN2-NEXT: v_mov_b32_e32 v4, v0
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB66_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB66_4
; GCN2-NEXT: .LBB66_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB66_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB66_2
; GCN2-NEXT: .LBB66_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_or_b32_e32 v2, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_or_b32_e32 v3, v1, v3
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v5, v1
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_mov_b32_e32 v4, v0
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB66_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB66_4
; GCN3-NEXT: .LBB66_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB66_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB66_2
; GCN3-NEXT: .LBB66_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_or_b32_e32 v3, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_or_b32_e32 v2, v0, v2
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw or ptr %ptr, i64 %in seq_cst
ret i64 %result
}
define i64 @flat_atomic_or_i64_ret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_or_i64_ret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB67_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB67_4
; GCN1-NEXT: .LBB67_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB67_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB67_2
; GCN1-NEXT: .LBB67_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_or_b32_e32 v2, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_or_b32_e32 v3, v1, v3
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i64_ret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB67_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB67_4
; GCN2-NEXT: .LBB67_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB67_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB67_2
; GCN2-NEXT: .LBB67_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_or_b32_e32 v2, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_or_b32_e32 v3, v1, v3
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i64_ret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB67_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB67_4
; GCN3-NEXT: .LBB67_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB67_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB67_2
; GCN3-NEXT: .LBB67_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_or_b32_e32 v3, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_or_b32_e32 v2, v0, v2
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw or ptr %gep, i64 %in seq_cst
ret i64 %result
}
define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_or_i64_noret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_mov_b64 s[34:35], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB68_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB68_4
; GCN1-NEXT: .LBB68_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB68_3: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execnz .LBB68_2
; GCN1-NEXT: .LBB68_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_or_b32_e32 v2, s6, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_or_b32_e32 v3, s7, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i64_noret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_mov_b64 s[34:35], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB68_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB68_4
; GCN2-NEXT: .LBB68_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB68_3: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execnz .LBB68_2
; GCN2-NEXT: .LBB68_4: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_or_b32_e32 v2, s6, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_or_b32_e32 v3, s7, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i64_noret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_mov_b64 s[34:35], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB68_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB68_4
; GCN3-NEXT: .LBB68_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB68_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execnz .LBB68_2
; GCN3-NEXT: .LBB68_4: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_or_b32_e32 v1, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_or_b32_e32 v2, s6, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst
ret void
}
define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_or_i64_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_mov_b64 s[36:37], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB69_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB69_4
; GCN1-NEXT: .LBB69_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB69_3: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execnz .LBB69_2
; GCN1-NEXT: .LBB69_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_or_b32_e32 v2, s6, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_or_b32_e32 v3, s7, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i64_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_mov_b64 s[36:37], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB69_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB69_4
; GCN2-NEXT: .LBB69_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB69_3: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execnz .LBB69_2
; GCN2-NEXT: .LBB69_4: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_or_b32_e32 v2, s6, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_or_b32_e32 v3, s7, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i64_noret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_mov_b64 s[36:37], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB69_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB69_4
; GCN3-NEXT: .LBB69_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB69_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s35
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execnz .LBB69_2
; GCN3-NEXT: .LBB69_4: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_or_b32_e32 v1, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_or_b32_e32 v2, s6, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst
ret void
}
define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_or_i64_ret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB70_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execz .LBB70_3
; GCN1-NEXT: s_branch .LBB70_4
; GCN1-NEXT: .LBB70_2:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: .LBB70_3: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_or_b32_e32 v4, s6, v0
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_or_b32_e32 v5, s7, v1
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB70_4: ; %atomicrmw.end
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i64_ret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB70_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execz .LBB70_3
; GCN2-NEXT: s_branch .LBB70_4
; GCN2-NEXT: .LBB70_2:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: .LBB70_3: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_or_b32_e32 v4, s6, v0
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_or_b32_e32 v5, s7, v1
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB70_4: ; %atomicrmw.end
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i64_ret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB70_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execz .LBB70_3
; GCN3-NEXT: s_branch .LBB70_4
; GCN3-NEXT: .LBB70_2:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: .LBB70_3: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_or_b32_e32 v3, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_or_b32_e32 v4, s6, v0
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB70_4: ; %atomicrmw.end
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw or ptr %ptr, i64 %in seq_cst
ret i64 %result
}
define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_or_i64_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB71_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execz .LBB71_3
; GCN1-NEXT: s_branch .LBB71_4
; GCN1-NEXT: .LBB71_2:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: .LBB71_3: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_or_b32_e32 v4, s6, v0
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_or_b32_e32 v5, s7, v1
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB71_4: ; %atomicrmw.end
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i64_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB71_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execz .LBB71_3
; GCN2-NEXT: s_branch .LBB71_4
; GCN2-NEXT: .LBB71_2:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: .LBB71_3: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_or_b32_e32 v4, s6, v0
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_or_b32_e32 v5, s7, v1
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB71_4: ; %atomicrmw.end
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i64_ret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB71_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s35
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execz .LBB71_3
; GCN3-NEXT: s_branch .LBB71_4
; GCN3-NEXT: .LBB71_2:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: .LBB71_3: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_or_b32_e32 v3, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_or_b32_e32 v4, s6, v0
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB71_4: ; %atomicrmw.end
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw or ptr %gep, i64 %in seq_cst
ret i64 %result
}
define void @flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB72_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB72_4
; GCN1-NEXT: .LBB72_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB72_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB72_2
; GCN1-NEXT: .LBB72_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_or_b32_e32 v2, v4, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_or_b32_e32 v3, v5, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB72_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB72_4
; GCN2-NEXT: .LBB72_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB72_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB72_2
; GCN2-NEXT: .LBB72_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_or_b32_e32 v2, v4, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_or_b32_e32 v3, v5, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB72_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB72_4
; GCN3-NEXT: .LBB72_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB72_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB72_2
; GCN3-NEXT: .LBB72_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_or_b32_e32 v1, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_or_b32_e32 v2, v4, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
ret void
}
define i64 @flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB73_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB73_4
; GCN1-NEXT: .LBB73_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB73_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB73_2
; GCN1-NEXT: .LBB73_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_or_b32_e32 v2, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_or_b32_e32 v3, v1, v3
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB73_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB73_4
; GCN2-NEXT: .LBB73_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB73_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB73_2
; GCN2-NEXT: .LBB73_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_or_b32_e32 v2, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_or_b32_e32 v3, v1, v3
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB73_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB73_4
; GCN3-NEXT: .LBB73_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB73_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB73_2
; GCN3-NEXT: .LBB73_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_or_b32_e32 v3, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_or_b32_e32 v2, v0, v2
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw or ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
ret i64 %result
}
; ---------------------------------------------------------------------
; atomicrmw xor
; ---------------------------------------------------------------------
define void @flat_atomic_xor_i64_noret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_xor_i64_noret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB74_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB74_4
; GCN1-NEXT: .LBB74_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB74_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB74_2
; GCN1-NEXT: .LBB74_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i64_noret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB74_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB74_4
; GCN2-NEXT: .LBB74_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB74_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB74_2
; GCN2-NEXT: .LBB74_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i64_noret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB74_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB74_4
; GCN3-NEXT: .LBB74_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB74_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB74_2
; GCN3-NEXT: .LBB74_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst
ret void
}
define void @flat_atomic_xor_i64_noret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_xor_i64_noret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB75_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB75_4
; GCN1-NEXT: .LBB75_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB75_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB75_2
; GCN1-NEXT: .LBB75_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i64_noret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB75_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB75_4
; GCN2-NEXT: .LBB75_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB75_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB75_2
; GCN2-NEXT: .LBB75_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i64_noret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB75_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB75_4
; GCN3-NEXT: .LBB75_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB75_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB75_2
; GCN3-NEXT: .LBB75_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst
ret void
}
define i64 @flat_atomic_xor_i64_ret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_xor_i64_ret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_mov_b32_e32 v5, v1
; GCN1-NEXT: v_mov_b32_e32 v4, v0
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB76_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB76_4
; GCN1-NEXT: .LBB76_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB76_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB76_2
; GCN1-NEXT: .LBB76_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_xor_b32_e32 v2, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_xor_b32_e32 v3, v1, v3
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i64_ret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_mov_b32_e32 v5, v1
; GCN2-NEXT: v_mov_b32_e32 v4, v0
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB76_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB76_4
; GCN2-NEXT: .LBB76_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB76_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB76_2
; GCN2-NEXT: .LBB76_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_xor_b32_e32 v2, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_xor_b32_e32 v3, v1, v3
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v5, v1
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_mov_b32_e32 v4, v0
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB76_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB76_4
; GCN3-NEXT: .LBB76_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB76_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB76_2
; GCN3-NEXT: .LBB76_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw xor ptr %ptr, i64 %in seq_cst
ret i64 %result
}
define i64 @flat_atomic_xor_i64_ret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_xor_i64_ret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB77_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB77_4
; GCN1-NEXT: .LBB77_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB77_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB77_2
; GCN1-NEXT: .LBB77_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_xor_b32_e32 v2, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_xor_b32_e32 v3, v1, v3
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i64_ret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB77_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB77_4
; GCN2-NEXT: .LBB77_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB77_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB77_2
; GCN2-NEXT: .LBB77_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_xor_b32_e32 v2, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_xor_b32_e32 v3, v1, v3
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i64_ret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB77_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB77_4
; GCN3-NEXT: .LBB77_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB77_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB77_2
; GCN3-NEXT: .LBB77_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw xor ptr %gep, i64 %in seq_cst
ret i64 %result
}
define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_xor_i64_noret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_mov_b64 s[34:35], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB78_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB78_4
; GCN1-NEXT: .LBB78_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB78_3: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execnz .LBB78_2
; GCN1-NEXT: .LBB78_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_xor_b32_e32 v2, s6, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_xor_b32_e32 v3, s7, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i64_noret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_mov_b64 s[34:35], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB78_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB78_4
; GCN2-NEXT: .LBB78_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB78_3: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execnz .LBB78_2
; GCN2-NEXT: .LBB78_4: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_xor_b32_e32 v2, s6, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_xor_b32_e32 v3, s7, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i64_noret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_mov_b64 s[34:35], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB78_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB78_4
; GCN3-NEXT: .LBB78_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB78_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execnz .LBB78_2
; GCN3-NEXT: .LBB78_4: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_xor_b32_e32 v1, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_xor_b32_e32 v2, s6, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst
ret void
}
define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_mov_b64 s[36:37], -1
; GCN1-NEXT: s_cbranch_vccnz .LBB79_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB79_4
; GCN1-NEXT: .LBB79_2: ; %atomicrmw.phi
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB79_3: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execnz .LBB79_2
; GCN1-NEXT: .LBB79_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s34
; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_xor_b32_e32 v2, s6, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_xor_b32_e32 v3, s7, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_mov_b64 s[36:37], -1
; GCN2-NEXT: s_cbranch_vccnz .LBB79_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB79_4
; GCN2-NEXT: .LBB79_2: ; %atomicrmw.phi
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB79_3: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execnz .LBB79_2
; GCN2-NEXT: .LBB79_4: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s34
; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_xor_b32_e32 v2, s6, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_xor_b32_e32 v3, s7, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_mov_b64 s[36:37], -1
; GCN3-NEXT: s_cbranch_vccnz .LBB79_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB79_4
; GCN3-NEXT: .LBB79_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB79_3: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s35
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execnz .LBB79_2
; GCN3-NEXT: .LBB79_4: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_xor_b32_e32 v1, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_xor_b32_e32 v2, s6, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst
ret void
}
define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_xor_i64_ret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB80_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execz .LBB80_3
; GCN1-NEXT: s_branch .LBB80_4
; GCN1-NEXT: .LBB80_2:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: .LBB80_3: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
; GCN1-NEXT: s_cselect_b32 s34, s4, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_xor_b32_e32 v4, s6, v0
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_xor_b32_e32 v5, s7, v1
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB80_4: ; %atomicrmw.end
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i64_ret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB80_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execz .LBB80_3
; GCN2-NEXT: s_branch .LBB80_4
; GCN2-NEXT: .LBB80_2:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: .LBB80_3: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN2-NEXT: s_cselect_b32 s34, s4, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_xor_b32_e32 v4, s6, v0
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_xor_b32_e32 v5, s7, v1
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB80_4: ; %atomicrmw.end
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i64_ret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB80_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execz .LBB80_3
; GCN3-NEXT: s_branch .LBB80_4
; GCN3-NEXT: .LBB80_2:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: .LBB80_3: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0
; GCN3-NEXT: s_cselect_b32 s34, s4, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_xor_b32_e32 v3, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_xor_b32_e32 v4, s6, v0
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB80_4: ; %atomicrmw.end
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw xor ptr %ptr, i64 %in seq_cst
ret i64 %result
}
define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB81_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s35
; GCN1-NEXT: v_mov_b32_e32 v3, s7
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_cbranch_execz .LBB81_3
; GCN1-NEXT: s_branch .LBB81_4
; GCN1-NEXT: .LBB81_2:
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: .LBB81_3: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
; GCN1-NEXT: s_cselect_b32 s34, s34, -1
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_add_i32 s34, s34, 4
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_xor_b32_e32 v4, s6, v0
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_xor_b32_e32 v5, s7, v1
; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN1-NEXT: .LBB81_4: ; %atomicrmw.end
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4
; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB81_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s35
; GCN2-NEXT: v_mov_b32_e32 v3, s7
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_cbranch_execz .LBB81_3
; GCN2-NEXT: s_branch .LBB81_4
; GCN2-NEXT: .LBB81_2:
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: .LBB81_3: ; %atomicrmw.private
; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN2-NEXT: s_cselect_b32 s34, s34, -1
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_add_i32 s34, s34, 4
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_xor_b32_e32 v4, s6, v0
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_xor_b32_e32 v5, s7, v1
; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen
; GCN2-NEXT: .LBB81_4: ; %atomicrmw.end
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: s_add_u32 s34, s4, 32
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB81_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s35
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_cbranch_execz .LBB81_3
; GCN3-NEXT: s_branch .LBB81_4
; GCN3-NEXT: .LBB81_2:
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: .LBB81_3: ; %atomicrmw.private
; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0
; GCN3-NEXT: s_cselect_b32 s34, s34, -1
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_xor_b32_e32 v3, s7, v1
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_xor_b32_e32 v4, s6, v0
; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
; GCN3-NEXT: .LBB81_4: ; %atomicrmw.end
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw xor ptr %gep, i64 %in seq_cst
ret i64 %result
}
define void @flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB82_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB82_4
; GCN1-NEXT: .LBB82_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB82_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB82_2
; GCN1-NEXT: .LBB82_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3
; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB82_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB82_4
; GCN2-NEXT: .LBB82_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB82_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB82_2
; GCN2-NEXT: .LBB82_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0
; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3
; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB82_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB82_4
; GCN3-NEXT: .LBB82_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB82_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB82_2
; GCN3-NEXT: .LBB82_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2
; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
ret void
}
define i64 @flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB83_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB83_4
; GCN1-NEXT: .LBB83_2: ; %atomicrmw.phi
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
; GCN1-NEXT: .LBB83_3: ; %atomicrmw.global
; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN1-NEXT: ; implicit-def: $vgpr3
; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN1-NEXT: s_cbranch_execz .LBB83_2
; GCN1-NEXT: .LBB83_4: ; %atomicrmw.private
; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4
; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN1-NEXT: s_waitcnt vmcnt(1)
; GCN1-NEXT: v_xor_b32_e32 v2, v0, v2
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: v_xor_b32_e32 v3, v1, v3
; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_waitcnt vmcnt(0)
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5
; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB83_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB83_4
; GCN2-NEXT: .LBB83_2: ; %atomicrmw.phi
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
; GCN2-NEXT: .LBB83_3: ; %atomicrmw.global
; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN2-NEXT: ; implicit-def: $vgpr3
; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN2-NEXT: s_cbranch_execz .LBB83_2
; GCN2-NEXT: .LBB83_4: ; %atomicrmw.private
; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4
; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
; GCN2-NEXT: s_waitcnt vmcnt(1)
; GCN2-NEXT: v_xor_b32_e32 v2, v0, v2
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: v_xor_b32_e32 v3, v1, v3
; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_waitcnt vmcnt(0)
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0
; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base
; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1
; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB83_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB83_4
; GCN3-NEXT: .LBB83_2: ; %atomicrmw.phi
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB83_3: ; %atomicrmw.global
; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5
; GCN3-NEXT: ; implicit-def: $vgpr3
; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GCN3-NEXT: s_cbranch_execz .LBB83_2
; GCN3-NEXT: .LBB83_4: ; %atomicrmw.private
; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc
; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
; GCN3-NEXT: s_waitcnt vmcnt(1)
; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2
; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen
; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_waitcnt vmcnt(0)
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw xor ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
ret i64 %result
}
; ---------------------------------------------------------------------
; atomicrmw max
; ---------------------------------------------------------------------
define void @flat_atomic_max_i64_noret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_max_i64_noret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v6, v[0:1]
; GCN1-NEXT: flat_load_dword v7, v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB84_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v5
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v6, v4
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB84_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i64_noret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v6, v[0:1]
; GCN2-NEXT: flat_load_dword v7, v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB84_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v5
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v6, v4
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB84_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i64_noret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB84_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB84_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define void @flat_atomic_max_i64_noret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_max_i64_noret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v7, v[0:1]
; GCN1-NEXT: flat_load_dword v6, v[8:9]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB85_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v1
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v6, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB85_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i64_noret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v7, v[0:1]
; GCN2-NEXT: flat_load_dword v6, v[8:9]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB85_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v1
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v6, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB85_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i64_noret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB85_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB85_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define i64 @flat_atomic_max_i64_ret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_max_i64_ret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v0
; GCN1-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v4, v[0:1]
; GCN1-NEXT: flat_load_dword v5, v[5:6]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB86_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v7, v5
; GCN1-NEXT: v_mov_b32_e32 v6, v4
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB86_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v0, v4
; GCN1-NEXT: v_mov_b32_e32 v1, v5
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i64_ret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v0
; GCN2-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v4, v[0:1]
; GCN2-NEXT: flat_load_dword v5, v[5:6]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB86_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v7, v5
; GCN2-NEXT: v_mov_b32_e32 v6, v4
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB86_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v0, v4
; GCN2-NEXT: v_mov_b32_e32 v1, v5
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB86_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB86_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v0, v4
; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define i64 @flat_atomic_max_i64_ret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_max_i64_ret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB87_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB87_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i64_ret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB87_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB87_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i64_ret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB87_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB87_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v0, v4
; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_max_i64_noret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: v_mov_b32_e32 v4, s35
; GCN1-NEXT: flat_load_dword v2, v[0:1]
; GCN1-NEXT: flat_load_dword v3, v[3:4]
; GCN1-NEXT: v_mov_b32_e32 v4, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v6, s7
; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: .LBB88_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB88_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i64_noret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: v_mov_b32_e32 v4, s35
; GCN2-NEXT: flat_load_dword v2, v[0:1]
; GCN2-NEXT: flat_load_dword v3, v[3:4]
; GCN2-NEXT: v_mov_b32_e32 v4, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v6, s7
; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: .LBB88_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB88_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i64_noret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB88_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB88_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_max_i64_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_add_u32 s36, s4, 36
; GCN1-NEXT: s_addc_u32 s37, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s36
; GCN1-NEXT: v_mov_b32_e32 v1, s37
; GCN1-NEXT: v_mov_b32_e32 v4, s34
; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v6, s7
; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB89_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB89_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i64_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_add_u32 s36, s4, 36
; GCN2-NEXT: s_addc_u32 s37, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s36
; GCN2-NEXT: v_mov_b32_e32 v1, s37
; GCN2-NEXT: v_mov_b32_e32 v4, s34
; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v6, s7
; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB89_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB89_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i64_noret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB89_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB89_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_max_i64_ret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: flat_load_dword v1, v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: .LBB90_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB90_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i64_ret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: flat_load_dword v1, v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: .LBB90_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB90_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i64_ret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v4, s7
; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB90_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v1
; GCN3-NEXT: v_mov_b32_e32 v8, v0
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB90_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_max_i64_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_add_u32 s36, s4, 36
; GCN1-NEXT: s_addc_u32 s37, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s36
; GCN1-NEXT: v_mov_b32_e32 v1, s37
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB91_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB91_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i64_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_add_u32 s36, s4, 36
; GCN2-NEXT: s_addc_u32 s37, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s36
; GCN2-NEXT: v_mov_b32_e32 v1, s37
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB91_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB91_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i64_ret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v4, s7
; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB91_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v1
; GCN3-NEXT: v_mov_b32_e32 v8, v0
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB91_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_max_i64_addr64_offset:
; GCN1: ; %bb.0: ; %entry
; GCN1-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0xd
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
; GCN1-NEXT: s_add_u32 s0, s0, s4
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 32
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v5, s1
; GCN1-NEXT: v_mov_b32_e32 v4, s0
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: v_mov_b32_e32 v6, s3
; GCN1-NEXT: v_mov_b32_e32 v7, s2
; GCN1-NEXT: .LBB92_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB92_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_max_i64_addr64_offset:
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
; GCN2-NEXT: s_add_u32 s0, s0, s4
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v5, s1
; GCN2-NEXT: v_mov_b32_e32 v4, s0
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: v_mov_b32_e32 v6, s3
; GCN2-NEXT: v_mov_b32_e32 v7, s2
; GCN2-NEXT: .LBB92_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB92_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_max_i64_addr64_offset:
; GCN3: ; %bb.0: ; %entry
; GCN3-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x34
; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 3
; GCN3-NEXT: s_add_u32 s0, s4, s0
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v5, s1
; GCN3-NEXT: v_mov_b32_e32 v4, s0
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB92_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB92_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
%gep = getelementptr i64, ptr %ptr, i64 4
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_max_i64_ret_addr64_offset:
; GCN1: ; %bb.0: ; %entry
; GCN1-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GCN1-NEXT: s_add_u32 s0, s0, s6
; GCN1-NEXT: s_addc_u32 s1, s1, s7
; GCN1-NEXT: s_add_u32 s0, s0, 32
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s5
; GCN1-NEXT: v_mov_b32_e32 v5, s4
; GCN1-NEXT: .LBB93_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v3
; GCN1-NEXT: v_mov_b32_e32 v8, v2
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB93_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN1-NEXT: v_mov_b32_e32 v0, s2
; GCN1-NEXT: v_mov_b32_e32 v1, s3
; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_max_i64_ret_addr64_offset:
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT: s_add_u32 s0, s0, s6
; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: v_mov_b32_e32 v4, s5
; GCN2-NEXT: v_mov_b32_e32 v5, s4
; GCN2-NEXT: .LBB93_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v3
; GCN2-NEXT: v_mov_b32_e32 v8, v2
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB93_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN2-NEXT: v_mov_b32_e32 v0, s2
; GCN2-NEXT: v_mov_b32_e32 v1, s3
; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_max_i64_ret_addr64_offset:
; GCN3: ; %bb.0: ; %entry
; GCN3-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_lshl_b64 s[0:1], s[10:11], 3
; GCN3-NEXT: s_add_u32 s0, s4, s0
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: v_mov_b32_e32 v4, s9
; GCN3-NEXT: v_mov_b32_e32 v5, s8
; GCN3-NEXT: .LBB93_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v3
; GCN3-NEXT: v_mov_b32_e32 v8, v2
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[8:9], v[8:9]
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB93_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
%gep = getelementptr i64, ptr %ptr, i64 4
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
store i64 %tmp0, ptr %out2
ret void
}
define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_max_i64_addr64:
; GCN1: ; %bb.0: ; %entry
; GCN1-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0xd
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
; GCN1-NEXT: s_add_u32 s0, s0, s4
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v5, s1
; GCN1-NEXT: v_mov_b32_e32 v4, s0
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: v_mov_b32_e32 v6, s3
; GCN1-NEXT: v_mov_b32_e32 v7, s2
; GCN1-NEXT: .LBB94_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB94_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_max_i64_addr64:
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
; GCN2-NEXT: s_add_u32 s0, s0, s4
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v5, s1
; GCN2-NEXT: v_mov_b32_e32 v4, s0
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: v_mov_b32_e32 v6, s3
; GCN2-NEXT: v_mov_b32_e32 v7, s2
; GCN2-NEXT: .LBB94_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB94_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_max_i64_addr64:
; GCN3: ; %bb.0: ; %entry
; GCN3-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x34
; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 3
; GCN3-NEXT: s_add_u32 s0, s4, s0
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v5, s1
; GCN3-NEXT: v_mov_b32_e32 v4, s0
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB94_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB94_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
%tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_max_i64_ret_addr64:
; GCN1: ; %bb.0: ; %entry
; GCN1-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GCN1-NEXT: s_add_u32 s0, s0, s6
; GCN1-NEXT: s_addc_u32 s1, s1, s7
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s5
; GCN1-NEXT: v_mov_b32_e32 v5, s4
; GCN1-NEXT: .LBB95_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v3
; GCN1-NEXT: v_mov_b32_e32 v8, v2
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB95_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN1-NEXT: v_mov_b32_e32 v0, s2
; GCN1-NEXT: v_mov_b32_e32 v1, s3
; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_max_i64_ret_addr64:
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT: s_add_u32 s0, s0, s6
; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: v_mov_b32_e32 v4, s5
; GCN2-NEXT: v_mov_b32_e32 v5, s4
; GCN2-NEXT: .LBB95_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v3
; GCN2-NEXT: v_mov_b32_e32 v8, v2
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB95_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN2-NEXT: v_mov_b32_e32 v0, s2
; GCN2-NEXT: v_mov_b32_e32 v1, s3
; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_max_i64_ret_addr64:
; GCN3: ; %bb.0: ; %entry
; GCN3-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_lshl_b64 s[0:1], s[10:11], 3
; GCN3-NEXT: s_add_u32 s0, s4, s0
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: v_mov_b32_e32 v4, s9
; GCN3-NEXT: v_mov_b32_e32 v5, s8
; GCN3-NEXT: .LBB95_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v3
; GCN3-NEXT: v_mov_b32_e32 v8, v2
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[8:9], v[8:9]
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB95_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
%tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
store i64 %tmp0, ptr %out2
ret void
}
define void @flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v7, v[0:1]
; GCN1-NEXT: flat_load_dword v6, v[8:9]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB96_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v1
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v6, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB96_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v7, v[0:1]
; GCN2-NEXT: flat_load_dword v6, v[8:9]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB96_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v1
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v6, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB96_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB96_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB96_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
ret void
}
define i64 @flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB97_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB97_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB97_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB97_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB97_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB97_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v0, v4
; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
ret i64 %result
}
; ---------------------------------------------------------------------
; atomicrmw umax
; ---------------------------------------------------------------------
define void @flat_atomic_umax_i64_noret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_umax_i64_noret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v6, v[0:1]
; GCN1-NEXT: flat_load_dword v7, v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB98_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v5
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v6, v4
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB98_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i64_noret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v6, v[0:1]
; GCN2-NEXT: flat_load_dword v7, v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB98_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v5
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v6, v4
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB98_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i64_noret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB98_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB98_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define void @flat_atomic_umax_i64_noret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_umax_i64_noret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v7, v[0:1]
; GCN1-NEXT: flat_load_dword v6, v[8:9]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB99_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v1
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v6, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB99_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i64_noret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v7, v[0:1]
; GCN2-NEXT: flat_load_dword v6, v[8:9]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB99_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v1
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v6, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB99_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i64_noret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB99_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB99_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define i64 @flat_atomic_umax_i64_ret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_umax_i64_ret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v0
; GCN1-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v4, v[0:1]
; GCN1-NEXT: flat_load_dword v5, v[5:6]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB100_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v7, v5
; GCN1-NEXT: v_mov_b32_e32 v6, v4
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB100_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v0, v4
; GCN1-NEXT: v_mov_b32_e32 v1, v5
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i64_ret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v0
; GCN2-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v4, v[0:1]
; GCN2-NEXT: flat_load_dword v5, v[5:6]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB100_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v7, v5
; GCN2-NEXT: v_mov_b32_e32 v6, v4
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB100_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v0, v4
; GCN2-NEXT: v_mov_b32_e32 v1, v5
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB100_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB100_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v0, v4
; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define i64 @flat_atomic_umax_i64_ret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_umax_i64_ret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB101_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB101_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i64_ret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB101_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB101_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i64_ret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB101_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB101_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v0, v4
; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_umax_i64_noret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: v_mov_b32_e32 v4, s35
; GCN1-NEXT: flat_load_dword v2, v[0:1]
; GCN1-NEXT: flat_load_dword v3, v[3:4]
; GCN1-NEXT: v_mov_b32_e32 v4, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v6, s7
; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: .LBB102_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB102_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i64_noret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: v_mov_b32_e32 v4, s35
; GCN2-NEXT: flat_load_dword v2, v[0:1]
; GCN2-NEXT: flat_load_dword v3, v[3:4]
; GCN2-NEXT: v_mov_b32_e32 v4, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v6, s7
; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: .LBB102_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB102_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i64_noret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB102_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB102_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_add_u32 s36, s4, 36
; GCN1-NEXT: s_addc_u32 s37, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s36
; GCN1-NEXT: v_mov_b32_e32 v1, s37
; GCN1-NEXT: v_mov_b32_e32 v4, s34
; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v6, s7
; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB103_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB103_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_add_u32 s36, s4, 36
; GCN2-NEXT: s_addc_u32 s37, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s36
; GCN2-NEXT: v_mov_b32_e32 v1, s37
; GCN2-NEXT: v_mov_b32_e32 v4, s34
; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v6, s7
; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB103_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB103_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB103_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB103_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_umax_i64_ret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: flat_load_dword v1, v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: .LBB104_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB104_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i64_ret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: flat_load_dword v1, v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: .LBB104_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB104_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i64_ret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v4, s7
; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB104_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v1
; GCN3-NEXT: v_mov_b32_e32 v8, v0
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB104_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_add_u32 s36, s4, 36
; GCN1-NEXT: s_addc_u32 s37, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s36
; GCN1-NEXT: v_mov_b32_e32 v1, s37
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB105_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB105_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_add_u32 s36, s4, 36
; GCN2-NEXT: s_addc_u32 s37, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s36
; GCN2-NEXT: v_mov_b32_e32 v1, s37
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB105_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB105_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v4, s7
; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB105_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v1
; GCN3-NEXT: v_mov_b32_e32 v8, v0
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB105_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umax_i64_addr64_offset:
; GCN1: ; %bb.0: ; %entry
; GCN1-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0xd
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
; GCN1-NEXT: s_add_u32 s0, s0, s4
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 32
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v5, s1
; GCN1-NEXT: v_mov_b32_e32 v4, s0
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: v_mov_b32_e32 v6, s3
; GCN1-NEXT: v_mov_b32_e32 v7, s2
; GCN1-NEXT: .LBB106_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB106_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_addr64_offset:
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
; GCN2-NEXT: s_add_u32 s0, s0, s4
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v5, s1
; GCN2-NEXT: v_mov_b32_e32 v4, s0
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: v_mov_b32_e32 v6, s3
; GCN2-NEXT: v_mov_b32_e32 v7, s2
; GCN2-NEXT: .LBB106_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB106_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_umax_i64_addr64_offset:
; GCN3: ; %bb.0: ; %entry
; GCN3-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x34
; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 3
; GCN3-NEXT: s_add_u32 s0, s4, s0
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v5, s1
; GCN3-NEXT: v_mov_b32_e32 v4, s0
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB106_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB106_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
%gep = getelementptr i64, ptr %ptr, i64 4
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umax_i64_ret_addr64_offset:
; GCN1: ; %bb.0: ; %entry
; GCN1-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GCN1-NEXT: s_add_u32 s0, s0, s6
; GCN1-NEXT: s_addc_u32 s1, s1, s7
; GCN1-NEXT: s_add_u32 s0, s0, 32
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s5
; GCN1-NEXT: v_mov_b32_e32 v5, s4
; GCN1-NEXT: .LBB107_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v3
; GCN1-NEXT: v_mov_b32_e32 v8, v2
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB107_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN1-NEXT: v_mov_b32_e32 v0, s2
; GCN1-NEXT: v_mov_b32_e32 v1, s3
; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_ret_addr64_offset:
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT: s_add_u32 s0, s0, s6
; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: v_mov_b32_e32 v4, s5
; GCN2-NEXT: v_mov_b32_e32 v5, s4
; GCN2-NEXT: .LBB107_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v3
; GCN2-NEXT: v_mov_b32_e32 v8, v2
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB107_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN2-NEXT: v_mov_b32_e32 v0, s2
; GCN2-NEXT: v_mov_b32_e32 v1, s3
; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_umax_i64_ret_addr64_offset:
; GCN3: ; %bb.0: ; %entry
; GCN3-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_lshl_b64 s[0:1], s[10:11], 3
; GCN3-NEXT: s_add_u32 s0, s4, s0
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: v_mov_b32_e32 v4, s9
; GCN3-NEXT: v_mov_b32_e32 v5, s8
; GCN3-NEXT: .LBB107_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v3
; GCN3-NEXT: v_mov_b32_e32 v8, v2
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[8:9], v[8:9]
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB107_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
%gep = getelementptr i64, ptr %ptr, i64 4
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
store i64 %tmp0, ptr %out2
ret void
}
define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_umax_i64_ret_addr64:
; GCN1: ; %bb.0: ; %entry
; GCN1-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GCN1-NEXT: s_add_u32 s0, s0, s6
; GCN1-NEXT: s_addc_u32 s1, s1, s7
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s5
; GCN1-NEXT: v_mov_b32_e32 v5, s4
; GCN1-NEXT: .LBB108_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v3
; GCN1-NEXT: v_mov_b32_e32 v8, v2
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB108_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN1-NEXT: v_mov_b32_e32 v0, s2
; GCN1-NEXT: v_mov_b32_e32 v1, s3
; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_ret_addr64:
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT: s_add_u32 s0, s0, s6
; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: v_mov_b32_e32 v4, s5
; GCN2-NEXT: v_mov_b32_e32 v5, s4
; GCN2-NEXT: .LBB108_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v3
; GCN2-NEXT: v_mov_b32_e32 v8, v2
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB108_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN2-NEXT: v_mov_b32_e32 v0, s2
; GCN2-NEXT: v_mov_b32_e32 v1, s3
; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_umax_i64_ret_addr64:
; GCN3: ; %bb.0: ; %entry
; GCN3-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_lshl_b64 s[0:1], s[10:11], 3
; GCN3-NEXT: s_add_u32 s0, s4, s0
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: v_mov_b32_e32 v4, s9
; GCN3-NEXT: v_mov_b32_e32 v5, s8
; GCN3-NEXT: .LBB108_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v3
; GCN3-NEXT: v_mov_b32_e32 v8, v2
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[8:9], v[8:9]
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB108_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
%tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
store i64 %tmp0, ptr %out2
ret void
}
define void @flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v7, v[0:1]
; GCN1-NEXT: flat_load_dword v6, v[8:9]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB109_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v1
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v6, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB109_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v7, v[0:1]
; GCN2-NEXT: flat_load_dword v6, v[8:9]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB109_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v1
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v6, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB109_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB109_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB109_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
ret void
}
define i64 @flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB110_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB110_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB110_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB110_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB110_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB110_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v0, v4
; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
ret i64 %result
}
; ---------------------------------------------------------------------
; atomicrmw umin
; ---------------------------------------------------------------------
define void @flat_atomic_umin_i64_noret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_umin_i64_noret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v6, v[0:1]
; GCN1-NEXT: flat_load_dword v7, v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB111_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v5
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v6, v4
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB111_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i64_noret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v6, v[0:1]
; GCN2-NEXT: flat_load_dword v7, v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB111_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v5
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v6, v4
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB111_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i64_noret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB111_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB111_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define void @flat_atomic_umin_i64_noret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_umin_i64_noret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v7, v[0:1]
; GCN1-NEXT: flat_load_dword v6, v[8:9]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB112_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v1
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v6, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB112_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i64_noret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v7, v[0:1]
; GCN2-NEXT: flat_load_dword v6, v[8:9]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB112_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v1
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v6, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB112_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i64_noret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB112_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB112_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define i64 @flat_atomic_umin_i64_ret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_umin_i64_ret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v0
; GCN1-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v4, v[0:1]
; GCN1-NEXT: flat_load_dword v5, v[5:6]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB113_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v7, v5
; GCN1-NEXT: v_mov_b32_e32 v6, v4
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB113_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v0, v4
; GCN1-NEXT: v_mov_b32_e32 v1, v5
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i64_ret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v0
; GCN2-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v4, v[0:1]
; GCN2-NEXT: flat_load_dword v5, v[5:6]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB113_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v7, v5
; GCN2-NEXT: v_mov_b32_e32 v6, v4
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB113_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v0, v4
; GCN2-NEXT: v_mov_b32_e32 v1, v5
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB113_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB113_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v0, v4
; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define i64 @flat_atomic_umin_i64_ret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_umin_i64_ret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB114_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB114_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i64_ret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB114_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB114_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i64_ret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB114_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB114_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v0, v4
; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_umin_i64_noret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: v_mov_b32_e32 v4, s35
; GCN1-NEXT: flat_load_dword v2, v[0:1]
; GCN1-NEXT: flat_load_dword v3, v[3:4]
; GCN1-NEXT: v_mov_b32_e32 v4, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v6, s7
; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: .LBB115_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB115_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i64_noret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: v_mov_b32_e32 v4, s35
; GCN2-NEXT: flat_load_dword v2, v[0:1]
; GCN2-NEXT: flat_load_dword v3, v[3:4]
; GCN2-NEXT: v_mov_b32_e32 v4, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v6, s7
; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: .LBB115_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB115_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i64_noret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB115_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB115_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_add_u32 s36, s4, 36
; GCN1-NEXT: s_addc_u32 s37, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s36
; GCN1-NEXT: v_mov_b32_e32 v1, s37
; GCN1-NEXT: v_mov_b32_e32 v4, s34
; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v6, s7
; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB116_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB116_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_add_u32 s36, s4, 36
; GCN2-NEXT: s_addc_u32 s37, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s36
; GCN2-NEXT: v_mov_b32_e32 v1, s37
; GCN2-NEXT: v_mov_b32_e32 v4, s34
; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v6, s7
; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB116_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB116_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB116_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB116_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_umin_i64_ret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: flat_load_dword v1, v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: .LBB117_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB117_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i64_ret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: flat_load_dword v1, v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: .LBB117_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB117_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i64_ret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v4, s7
; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB117_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v1
; GCN3-NEXT: v_mov_b32_e32 v8, v0
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB117_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_add_u32 s36, s4, 36
; GCN1-NEXT: s_addc_u32 s37, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s36
; GCN1-NEXT: v_mov_b32_e32 v1, s37
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB118_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB118_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_add_u32 s36, s4, 36
; GCN2-NEXT: s_addc_u32 s37, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s36
; GCN2-NEXT: v_mov_b32_e32 v1, s37
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB118_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB118_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v4, s7
; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB118_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v1
; GCN3-NEXT: v_mov_b32_e32 v8, v0
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB118_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define void @flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v7, v[0:1]
; GCN1-NEXT: flat_load_dword v6, v[8:9]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB119_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v1
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v6, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB119_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v7, v[0:1]
; GCN2-NEXT: flat_load_dword v6, v[8:9]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB119_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v1
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v6, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB119_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB119_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB119_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
ret void
}
define i64 @flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB120_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB120_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB120_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB120_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB120_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB120_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v0, v4
; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
ret i64 %result
}
; ---------------------------------------------------------------------
; atomicrmw min
; ---------------------------------------------------------------------
define void @flat_atomic_min_i64_noret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_min_i64_noret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v6, v[0:1]
; GCN1-NEXT: flat_load_dword v7, v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB121_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v5
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v6, v4
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB121_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i64_noret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v6, v[0:1]
; GCN2-NEXT: flat_load_dword v7, v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB121_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v5
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v6, v4
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB121_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i64_noret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB121_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB121_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define void @flat_atomic_min_i64_noret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_min_i64_noret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v7, v[0:1]
; GCN1-NEXT: flat_load_dword v6, v[8:9]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB122_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v1
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v6, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB122_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i64_noret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v7, v[0:1]
; GCN2-NEXT: flat_load_dword v6, v[8:9]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB122_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v1
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v6, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB122_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i64_noret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB122_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB122_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define i64 @flat_atomic_min_i64_ret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_min_i64_ret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v0
; GCN1-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v4, v[0:1]
; GCN1-NEXT: flat_load_dword v5, v[5:6]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB123_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v7, v5
; GCN1-NEXT: v_mov_b32_e32 v6, v4
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB123_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v0, v4
; GCN1-NEXT: v_mov_b32_e32 v1, v5
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i64_ret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v0
; GCN2-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v4, v[0:1]
; GCN2-NEXT: flat_load_dword v5, v[5:6]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB123_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v7, v5
; GCN2-NEXT: v_mov_b32_e32 v6, v4
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB123_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v0, v4
; GCN2-NEXT: v_mov_b32_e32 v1, v5
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB123_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB123_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v0, v4
; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define i64 @flat_atomic_min_i64_ret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_min_i64_ret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB124_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB124_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i64_ret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB124_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB124_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i64_ret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB124_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB124_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v0, v4
; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_min_i64_noret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v3, s34
; GCN1-NEXT: v_mov_b32_e32 v4, s35
; GCN1-NEXT: flat_load_dword v2, v[0:1]
; GCN1-NEXT: flat_load_dword v3, v[3:4]
; GCN1-NEXT: v_mov_b32_e32 v4, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v6, s7
; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: .LBB125_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB125_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i64_noret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v3, s34
; GCN2-NEXT: v_mov_b32_e32 v4, s35
; GCN2-NEXT: flat_load_dword v2, v[0:1]
; GCN2-NEXT: flat_load_dword v3, v[3:4]
; GCN2-NEXT: v_mov_b32_e32 v4, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v6, s7
; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: .LBB125_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB125_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i64_noret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB125_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB125_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_min_i64_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_add_u32 s36, s4, 36
; GCN1-NEXT: s_addc_u32 s37, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s36
; GCN1-NEXT: v_mov_b32_e32 v1, s37
; GCN1-NEXT: v_mov_b32_e32 v4, s34
; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v6, s7
; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB126_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB126_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i64_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_add_u32 s36, s4, 36
; GCN2-NEXT: s_addc_u32 s37, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s36
; GCN2-NEXT: v_mov_b32_e32 v1, s37
; GCN2-NEXT: v_mov_b32_e32 v4, s34
; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v6, s7
; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB126_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB126_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i64_noret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB126_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB126_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_min_i64_ret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: flat_load_dword v1, v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: .LBB127_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB127_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i64_ret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: flat_load_dword v1, v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: .LBB127_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB127_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i64_ret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v4, s7
; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB127_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v1
; GCN3-NEXT: v_mov_b32_e32 v8, v0
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB127_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_min_i64_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: s_add_u32 s36, s4, 36
; GCN1-NEXT: s_addc_u32 s37, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s36
; GCN1-NEXT: v_mov_b32_e32 v1, s37
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB128_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB128_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i64_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: s_add_u32 s36, s4, 36
; GCN2-NEXT: s_addc_u32 s37, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s36
; GCN2-NEXT: v_mov_b32_e32 v1, s37
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB128_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB128_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i64_ret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: v_mov_b32_e32 v4, s7
; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB128_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v1
; GCN3-NEXT: v_mov_b32_e32 v8, v0
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB128_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_min_i64_addr64_offset:
; GCN1: ; %bb.0: ; %entry
; GCN1-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0xd
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
; GCN1-NEXT: s_add_u32 s0, s0, s4
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 32
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v5, s1
; GCN1-NEXT: v_mov_b32_e32 v4, s0
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: v_mov_b32_e32 v6, s3
; GCN1-NEXT: v_mov_b32_e32 v7, s2
; GCN1-NEXT: .LBB129_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB129_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_min_i64_addr64_offset:
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
; GCN2-NEXT: s_add_u32 s0, s0, s4
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v5, s1
; GCN2-NEXT: v_mov_b32_e32 v4, s0
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: v_mov_b32_e32 v6, s3
; GCN2-NEXT: v_mov_b32_e32 v7, s2
; GCN2-NEXT: .LBB129_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB129_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_min_i64_addr64_offset:
; GCN3: ; %bb.0: ; %entry
; GCN3-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x34
; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 3
; GCN3-NEXT: s_add_u32 s0, s4, s0
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v5, s1
; GCN3-NEXT: v_mov_b32_e32 v4, s0
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB129_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB129_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
%gep = getelementptr i64, ptr %ptr, i64 4
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_min_i64_ret_addr64_offset:
; GCN1: ; %bb.0: ; %entry
; GCN1-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GCN1-NEXT: s_add_u32 s0, s0, s6
; GCN1-NEXT: s_addc_u32 s1, s1, s7
; GCN1-NEXT: s_add_u32 s0, s0, 32
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s5
; GCN1-NEXT: v_mov_b32_e32 v5, s4
; GCN1-NEXT: .LBB130_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v3
; GCN1-NEXT: v_mov_b32_e32 v8, v2
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB130_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN1-NEXT: v_mov_b32_e32 v0, s2
; GCN1-NEXT: v_mov_b32_e32 v1, s3
; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_min_i64_ret_addr64_offset:
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT: s_add_u32 s0, s0, s6
; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: v_mov_b32_e32 v4, s5
; GCN2-NEXT: v_mov_b32_e32 v5, s4
; GCN2-NEXT: .LBB130_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v3
; GCN2-NEXT: v_mov_b32_e32 v8, v2
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB130_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN2-NEXT: v_mov_b32_e32 v0, s2
; GCN2-NEXT: v_mov_b32_e32 v1, s3
; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_min_i64_ret_addr64_offset:
; GCN3: ; %bb.0: ; %entry
; GCN3-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_lshl_b64 s[0:1], s[10:11], 3
; GCN3-NEXT: s_add_u32 s0, s4, s0
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: v_mov_b32_e32 v4, s9
; GCN3-NEXT: v_mov_b32_e32 v5, s8
; GCN3-NEXT: .LBB130_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v3
; GCN3-NEXT: v_mov_b32_e32 v8, v2
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[8:9], v[8:9]
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB130_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
%gep = getelementptr i64, ptr %ptr, i64 4
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
store i64 %tmp0, ptr %out2
ret void
}
define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GCN1-LABEL: atomic_min_i64:
; GCN1: ; %bb.0: ; %entry
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x9
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT: v_mov_b32_e32 v5, s1
; GCN1-NEXT: v_mov_b32_e32 v6, s3
; GCN1-NEXT: v_mov_b32_e32 v7, s2
; GCN1-NEXT: v_mov_b32_e32 v4, s0
; GCN1-NEXT: .LBB131_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB131_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_min_i64:
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT: v_mov_b32_e32 v5, s1
; GCN2-NEXT: v_mov_b32_e32 v6, s3
; GCN2-NEXT: v_mov_b32_e32 v7, s2
; GCN2-NEXT: v_mov_b32_e32 v4, s0
; GCN2-NEXT: .LBB131_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB131_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_min_i64:
; GCN3: ; %bb.0: ; %entry
; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB131_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB131_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
entry:
%tmp0 = atomicrmw min ptr %out, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
; GCN1-LABEL: atomic_min_i64_ret_addr64:
; GCN1: ; %bb.0: ; %entry
; GCN1-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GCN1-NEXT: s_add_u32 s0, s0, s6
; GCN1-NEXT: s_addc_u32 s1, s1, s7
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: v_mov_b32_e32 v4, s5
; GCN1-NEXT: v_mov_b32_e32 v5, s4
; GCN1-NEXT: .LBB132_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v3
; GCN1-NEXT: v_mov_b32_e32 v8, v2
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB132_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN1-NEXT: v_mov_b32_e32 v0, s2
; GCN1-NEXT: v_mov_b32_e32 v1, s3
; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_min_i64_ret_addr64:
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GCN2-NEXT: s_add_u32 s0, s0, s6
; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: v_mov_b32_e32 v4, s5
; GCN2-NEXT: v_mov_b32_e32 v5, s4
; GCN2-NEXT: .LBB132_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v3
; GCN2-NEXT: v_mov_b32_e32 v8, v2
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB132_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN2-NEXT: v_mov_b32_e32 v0, s2
; GCN2-NEXT: v_mov_b32_e32 v1, s3
; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_min_i64_ret_addr64:
; GCN3: ; %bb.0: ; %entry
; GCN3-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_lshl_b64 s[0:1], s[10:11], 3
; GCN3-NEXT: s_add_u32 s0, s4, s0
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: v_mov_b32_e32 v4, s9
; GCN3-NEXT: v_mov_b32_e32 v5, s8
; GCN3-NEXT: .LBB132_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v9, v3
; GCN3-NEXT: v_mov_b32_e32 v8, v2
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[8:9], v[8:9]
; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB132_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
%tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
store i64 %tmp0, ptr %out2
ret void
}
define void @flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v8, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v7, v[0:1]
; GCN1-NEXT: flat_load_dword v6, v[8:9]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB133_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: v_mov_b32_e32 v7, v1
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v6, v0
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB133_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v8, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v7, v[0:1]
; GCN2-NEXT: flat_load_dword v6, v[8:9]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB133_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: v_mov_b32_e32 v7, v1
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v6, v0
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB133_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB133_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB133_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
ret void
}
define i64 @flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 36, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: .LBB134_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v9, v1
; GCN1-NEXT: v_mov_b32_e32 v8, v0
; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB134_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 36, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: .LBB134_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v9, v1
; GCN2-NEXT: v_mov_b32_e32 v8, v0
; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB134_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:32
; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: .LBB134_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v7, v5
; GCN3-NEXT: v_mov_b32_e32 v6, v4
; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB134_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v0, v4
; GCN3-NEXT: v_mov_b32_e32 v1, v5
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
ret i64 %result
}
; ---------------------------------------------------------------------
; atomicrmw uinc_wrap
; ---------------------------------------------------------------------
define void @flat_atomic_uinc_wrap_i64_noret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define void @flat_atomic_uinc_wrap_i64_noret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] offset:32
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define i64 @flat_atomic_uinc_wrap_i64_ret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define i64 @flat_atomic_uinc_wrap_i64_ret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s7
; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s7
; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: v_mov_b32_e32 v0, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s7
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: v_mov_b32_e32 v0, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s7
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1] offset:32
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s7
; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s7
; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: v_mov_b32_e32 v0, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s7
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: v_mov_b32_e32 v0, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s7
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define void @flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] offset:32
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
ret void
}
define i64 @flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
ret i64 %result
}
; ---------------------------------------------------------------------
; atomicrmw udec_wrap
; ---------------------------------------------------------------------
define void @flat_atomic_udec_wrap_i64_noret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define void @flat_atomic_udec_wrap_i64_noret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] offset:32
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define i64 @flat_atomic_udec_wrap_i64_ret(ptr %ptr, i64 %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define i64 @flat_atomic_udec_wrap_i64_ret_offset(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s7
; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s7
; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: v_mov_b32_e32 v0, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s7
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: v_mov_b32_e32 v0, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s7
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1] offset:32
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret void
}
define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s7
; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s7
; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 32
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: v_mov_b32_e32 v0, s6
; GCN1-NEXT: v_mov_b32_e32 v1, s7
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 32
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: v_mov_b32_e32 v0, s6
; GCN2-NEXT: v_mov_b32_e32 v1, s7
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
ret i64 %result
}
define void @flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] offset:32
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
ret void
}
define i64 @flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:
; GCN3: ; %bb.0:
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr i64, ptr %out, i64 4
%result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1, !amdgpu.no.remote.memory !0
ret i64 %result
}
!0 = !{}
!1 = !{i32 5, i32 6}