| ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py |
| ; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefix=GCN1 %s |
| ; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=GCN2 %s |
| ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN3 %s |
| |
| ; --------------------------------------------------------------------- |
| ; atomicrmw xchg |
| ; --------------------------------------------------------------------- |
| |
| define void @flat_atomic_xchg_i64_noret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_i64_noret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB0_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB0_4 |
| ; GCN1-NEXT: .LBB0_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB0_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB0_2 |
| ; GCN1-NEXT: .LBB0_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_i64_noret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB0_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB0_4 |
| ; GCN2-NEXT: .LBB0_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB0_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB0_2 |
| ; GCN2-NEXT: .LBB0_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_i64_noret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB0_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB0_4 |
| ; GCN3-NEXT: .LBB0_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB0_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB0_2 |
| ; GCN3-NEXT: .LBB0_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw xchg ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define void @flat_atomic_xchg_i64_noret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB1_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB1_4 |
| ; GCN1-NEXT: .LBB1_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB1_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB1_2 |
| ; GCN1-NEXT: .LBB1_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB1_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB1_4 |
| ; GCN2-NEXT: .LBB1_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB1_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB1_2 |
| ; GCN2-NEXT: .LBB1_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB1_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB1_4 |
| ; GCN3-NEXT: .LBB1_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB1_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB1_2 |
| ; GCN3-NEXT: .LBB1_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define i64 @flat_atomic_xchg_i64_ret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_i64_ret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB2_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB2_4 |
| ; GCN1-NEXT: .LBB2_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB2_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB2_2 |
| ; GCN1-NEXT: .LBB2_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_i64_ret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB2_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB2_4 |
| ; GCN2-NEXT: .LBB2_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB2_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB2_2 |
| ; GCN2-NEXT: .LBB2_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_i64_ret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB2_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB2_4 |
| ; GCN3-NEXT: .LBB2_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB2_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB2_2 |
| ; GCN3-NEXT: .LBB2_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_nop 0 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw xchg ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define i64 @flat_atomic_xchg_i64_ret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB3_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB3_4 |
| ; GCN1-NEXT: .LBB3_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB3_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB3_2 |
| ; GCN1-NEXT: .LBB3_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB3_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB3_4 |
| ; GCN2-NEXT: .LBB3_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB3_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB3_2 |
| ; GCN2-NEXT: .LBB3_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB3_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB3_4 |
| ; GCN3-NEXT: .LBB3_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB3_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB3_2 |
| ; GCN3-NEXT: .LBB3_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_nop 0 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw xchg ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx void @flat_atomic_xchg_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_i64_noret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB4_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB4_4 |
| ; GCN1-NEXT: .LBB4_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB4_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB4_2 |
| ; GCN1-NEXT: .LBB4_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s6 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_i64_noret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB4_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB4_4 |
| ; GCN2-NEXT: .LBB4_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB4_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB4_2 |
| ; GCN2-NEXT: .LBB4_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_i64_noret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB4_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB4_4 |
| ; GCN3-NEXT: .LBB4_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB4_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB4_2 |
| ; GCN3-NEXT: .LBB4_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s6 |
| ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw xchg ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx void @flat_atomic_xchg_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB5_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB5_4 |
| ; GCN1-NEXT: .LBB5_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB5_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB5_2 |
| ; GCN1-NEXT: .LBB5_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s6 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB5_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB5_4 |
| ; GCN2-NEXT: .LBB5_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB5_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB5_2 |
| ; GCN2-NEXT: .LBB5_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB5_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB5_4 |
| ; GCN3-NEXT: .LBB5_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB5_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB5_2 |
| ; GCN3-NEXT: .LBB5_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s6 |
| ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_i64_ret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB6_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB6_3 |
| ; GCN1-NEXT: s_branch .LBB6_4 |
| ; GCN1-NEXT: .LBB6_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB6_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s7 |
| ; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB6_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_i64_ret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB6_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB6_3 |
| ; GCN2-NEXT: s_branch .LBB6_4 |
| ; GCN2-NEXT: .LBB6_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB6_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s7 |
| ; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB6_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_i64_ret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB6_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB6_3 |
| ; GCN3-NEXT: s_branch .LBB6_4 |
| ; GCN3-NEXT: .LBB6_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB6_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s6 |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB6_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw xchg ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB7_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB7_3 |
| ; GCN1-NEXT: s_branch .LBB7_4 |
| ; GCN1-NEXT: .LBB7_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB7_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s7 |
| ; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB7_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB7_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB7_3 |
| ; GCN2-NEXT: s_branch .LBB7_4 |
| ; GCN2-NEXT: .LBB7_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB7_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s7 |
| ; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB7_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB7_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB7_3 |
| ; GCN3-NEXT: s_branch .LBB7_4 |
| ; GCN3-NEXT: .LBB7_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB7_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s6 |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB7_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw xchg ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define void @flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB8_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB8_4 |
| ; GCN1-NEXT: .LBB8_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB8_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB8_2 |
| ; GCN1-NEXT: .LBB8_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB8_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB8_4 |
| ; GCN2-NEXT: .LBB8_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB8_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB8_2 |
| ; GCN2-NEXT: .LBB8_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB8_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB8_4 |
| ; GCN3-NEXT: .LBB8_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB8_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB8_2 |
| ; GCN3-NEXT: .LBB8_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret void |
| } |
| |
| define i64 @flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB9_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB9_4 |
| ; GCN1-NEXT: .LBB9_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB9_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB9_2 |
| ; GCN1-NEXT: .LBB9_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB9_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB9_4 |
| ; GCN2-NEXT: .LBB9_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB9_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB9_2 |
| ; GCN2-NEXT: .LBB9_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB9_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB9_4 |
| ; GCN3-NEXT: .LBB9_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB9_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB9_2 |
| ; GCN3-NEXT: .LBB9_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_nop 0 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw xchg ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret i64 %result |
| } |
| |
| ; --------------------------------------------------------------------- |
| ; atomicrmw xchg f64 |
| ; --------------------------------------------------------------------- |
| |
| define void @flat_atomic_xchg_f64_noret(ptr %ptr, double %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_f64_noret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB10_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB10_4 |
| ; GCN1-NEXT: .LBB10_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB10_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB10_2 |
| ; GCN1-NEXT: .LBB10_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_f64_noret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB10_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB10_4 |
| ; GCN2-NEXT: .LBB10_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB10_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB10_2 |
| ; GCN2-NEXT: .LBB10_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_f64_noret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB10_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB10_4 |
| ; GCN3-NEXT: .LBB10_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB10_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB10_2 |
| ; GCN3-NEXT: .LBB10_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw xchg ptr %ptr, double %in seq_cst |
| ret void |
| } |
| |
| define void @flat_atomic_xchg_f64_noret_offset(ptr %out, double %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB11_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB11_4 |
| ; GCN1-NEXT: .LBB11_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB11_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB11_2 |
| ; GCN1-NEXT: .LBB11_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB11_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB11_4 |
| ; GCN2-NEXT: .LBB11_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB11_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB11_2 |
| ; GCN2-NEXT: .LBB11_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB11_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB11_4 |
| ; GCN3-NEXT: .LBB11_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB11_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB11_2 |
| ; GCN3-NEXT: .LBB11_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr double, ptr %out, i32 4 |
| %tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst |
| ret void |
| } |
| |
| define double @flat_atomic_xchg_f64_ret(ptr %ptr, double %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_f64_ret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB12_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB12_4 |
| ; GCN1-NEXT: .LBB12_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB12_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB12_2 |
| ; GCN1-NEXT: .LBB12_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_f64_ret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB12_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB12_4 |
| ; GCN2-NEXT: .LBB12_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB12_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB12_2 |
| ; GCN2-NEXT: .LBB12_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_f64_ret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB12_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB12_4 |
| ; GCN3-NEXT: .LBB12_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB12_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB12_2 |
| ; GCN3-NEXT: .LBB12_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_nop 0 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw xchg ptr %ptr, double %in seq_cst |
| ret double %result |
| } |
| |
| define double @flat_atomic_xchg_f64_ret_offset(ptr %out, double %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB13_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB13_4 |
| ; GCN1-NEXT: .LBB13_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB13_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB13_2 |
| ; GCN1-NEXT: .LBB13_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB13_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB13_4 |
| ; GCN2-NEXT: .LBB13_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB13_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB13_2 |
| ; GCN2-NEXT: .LBB13_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB13_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB13_4 |
| ; GCN3-NEXT: .LBB13_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB13_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB13_2 |
| ; GCN3-NEXT: .LBB13_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_nop 0 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr double, ptr %out, i32 4 |
| %result = atomicrmw xchg ptr %gep, double %in seq_cst |
| ret double %result |
| } |
| |
| define amdgpu_gfx void @flat_atomic_xchg_f64_noret_scalar(ptr inreg %ptr, double inreg %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_f64_noret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB14_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB14_4 |
| ; GCN1-NEXT: .LBB14_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB14_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB14_2 |
| ; GCN1-NEXT: .LBB14_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s6 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_f64_noret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB14_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB14_4 |
| ; GCN2-NEXT: .LBB14_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB14_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB14_2 |
| ; GCN2-NEXT: .LBB14_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_f64_noret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB14_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB14_4 |
| ; GCN3-NEXT: .LBB14_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB14_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB14_2 |
| ; GCN3-NEXT: .LBB14_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s6 |
| ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw xchg ptr %ptr, double %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx void @flat_atomic_xchg_f64_noret_offset_scalar(ptr inreg %out, double inreg %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB15_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB15_4 |
| ; GCN1-NEXT: .LBB15_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB15_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB15_2 |
| ; GCN1-NEXT: .LBB15_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s6 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB15_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB15_4 |
| ; GCN2-NEXT: .LBB15_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB15_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB15_2 |
| ; GCN2-NEXT: .LBB15_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB15_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB15_4 |
| ; GCN3-NEXT: .LBB15_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB15_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB15_2 |
| ; GCN3-NEXT: .LBB15_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s6 |
| ; GCN3-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr double, ptr %out, i32 4 |
| %tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx double @flat_atomic_xchg_f64_ret_scalar(ptr inreg %ptr, double inreg %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_f64_ret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB16_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB16_3 |
| ; GCN1-NEXT: s_branch .LBB16_4 |
| ; GCN1-NEXT: .LBB16_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB16_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s7 |
| ; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB16_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_f64_ret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB16_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB16_3 |
| ; GCN2-NEXT: s_branch .LBB16_4 |
| ; GCN2-NEXT: .LBB16_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB16_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s7 |
| ; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB16_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_f64_ret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB16_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB16_3 |
| ; GCN3-NEXT: s_branch .LBB16_4 |
| ; GCN3-NEXT: .LBB16_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB16_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s6 |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB16_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw xchg ptr %ptr, double %in seq_cst |
| ret double %result |
| } |
| |
| define amdgpu_gfx double @flat_atomic_xchg_f64_ret_offset_scalar(ptr inreg %out, double inreg %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB17_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB17_3 |
| ; GCN1-NEXT: s_branch .LBB17_4 |
| ; GCN1-NEXT: .LBB17_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB17_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s7 |
| ; GCN1-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB17_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB17_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB17_3 |
| ; GCN2-NEXT: s_branch .LBB17_4 |
| ; GCN2-NEXT: .LBB17_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB17_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s7 |
| ; GCN2-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB17_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB17_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB17_3 |
| ; GCN3-NEXT: s_branch .LBB17_4 |
| ; GCN3-NEXT: .LBB17_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB17_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s6 |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB17_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr double, ptr %out, i32 4 |
| %result = atomicrmw xchg ptr %gep, double %in seq_cst |
| ret double %result |
| } |
| |
| define void @flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory(ptr %out, double %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB18_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB18_4 |
| ; GCN1-NEXT: .LBB18_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB18_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB18_2 |
| ; GCN1-NEXT: .LBB18_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB18_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB18_4 |
| ; GCN2-NEXT: .LBB18_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB18_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB18_2 |
| ; GCN2-NEXT: .LBB18_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB18_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB18_4 |
| ; GCN3-NEXT: .LBB18_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB18_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB18_2 |
| ; GCN3-NEXT: .LBB18_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr double, ptr %out, i64 4 |
| %tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret void |
| } |
| |
| define double @flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory(ptr %out, double %in) { |
| ; GCN1-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB19_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB19_4 |
| ; GCN1-NEXT: .LBB19_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB19_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB19_2 |
| ; GCN1-NEXT: .LBB19_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB19_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB19_4 |
| ; GCN2-NEXT: .LBB19_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB19_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB19_2 |
| ; GCN2-NEXT: .LBB19_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB19_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB19_4 |
| ; GCN3-NEXT: .LBB19_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB19_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB19_2 |
| ; GCN3-NEXT: .LBB19_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_nop 0 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr double, ptr %out, i64 4 |
| %result = atomicrmw xchg ptr %gep, double %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret double %result |
| } |
| |
| ; --------------------------------------------------------------------- |
| ; atomicrmw add |
| ; --------------------------------------------------------------------- |
| |
| define void @flat_atomic_add_i64_noret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_add_i64_noret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB20_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB20_4 |
| ; GCN1-NEXT: .LBB20_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB20_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB20_2 |
| ; GCN1-NEXT: .LBB20_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_add_i64_noret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB20_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB20_4 |
| ; GCN2-NEXT: .LBB20_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB20_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB20_2 |
| ; GCN2-NEXT: .LBB20_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_add_i64_noret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB20_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB20_4 |
| ; GCN3-NEXT: .LBB20_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB20_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB20_2 |
| ; GCN3-NEXT: .LBB20_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw add ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define void @flat_atomic_add_i64_noret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_add_i64_noret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB21_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB21_4 |
| ; GCN1-NEXT: .LBB21_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB21_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB21_2 |
| ; GCN1-NEXT: .LBB21_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_add_i64_noret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB21_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB21_4 |
| ; GCN2-NEXT: .LBB21_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB21_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB21_2 |
| ; GCN2-NEXT: .LBB21_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_add_i64_noret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB21_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB21_4 |
| ; GCN3-NEXT: .LBB21_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB21_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB21_2 |
| ; GCN3-NEXT: .LBB21_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define i64 @flat_atomic_add_i64_ret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_add_i64_ret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB22_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB22_4 |
| ; GCN1-NEXT: .LBB22_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB22_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB22_2 |
| ; GCN1-NEXT: .LBB22_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_add_i64_ret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB22_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB22_4 |
| ; GCN2-NEXT: .LBB22_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB22_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB22_2 |
| ; GCN2-NEXT: .LBB22_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_add_i64_ret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB22_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB22_4 |
| ; GCN3-NEXT: .LBB22_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB22_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB22_2 |
| ; GCN3-NEXT: .LBB22_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw add ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define i64 @flat_atomic_add_i64_ret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_add_i64_ret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB23_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB23_4 |
| ; GCN1-NEXT: .LBB23_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB23_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB23_2 |
| ; GCN1-NEXT: .LBB23_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_add_i64_ret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB23_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB23_4 |
| ; GCN2-NEXT: .LBB23_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB23_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB23_2 |
| ; GCN2-NEXT: .LBB23_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_add_i64_ret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB23_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB23_4 |
| ; GCN3-NEXT: .LBB23_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB23_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB23_2 |
| ; GCN3-NEXT: .LBB23_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw add ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx void @flat_atomic_add_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_add_i64_noret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB24_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB24_4 |
| ; GCN1-NEXT: .LBB24_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB24_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB24_2 |
| ; GCN1-NEXT: .LBB24_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, s6, v1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_add_i64_noret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB24_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB24_4 |
| ; GCN2-NEXT: .LBB24_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB24_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB24_2 |
| ; GCN2-NEXT: .LBB24_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, s6, v1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_add_i64_noret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB24_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB24_4 |
| ; GCN3-NEXT: .LBB24_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB24_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB24_2 |
| ; GCN3-NEXT: .LBB24_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, s6, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw add ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx void @flat_atomic_add_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_add_i64_noret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB25_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB25_4 |
| ; GCN1-NEXT: .LBB25_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB25_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB25_2 |
| ; GCN1-NEXT: .LBB25_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, s6, v1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_add_i64_noret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB25_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB25_4 |
| ; GCN2-NEXT: .LBB25_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB25_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB25_2 |
| ; GCN2-NEXT: .LBB25_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, s6, v1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v3, v4, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_add_i64_noret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB25_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB25_4 |
| ; GCN3-NEXT: .LBB25_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB25_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB25_2 |
| ; GCN3-NEXT: .LBB25_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, s6, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_add_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_add_i64_ret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB26_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB26_3 |
| ; GCN1-NEXT: s_branch .LBB26_4 |
| ; GCN1-NEXT: .LBB26_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB26_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, s6, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB26_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_add_i64_ret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB26_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB26_3 |
| ; GCN2-NEXT: s_branch .LBB26_4 |
| ; GCN2-NEXT: .LBB26_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB26_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, s6, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB26_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_add_i64_ret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB26_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB26_3 |
| ; GCN3-NEXT: s_branch .LBB26_4 |
| ; GCN3-NEXT: .LBB26_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB26_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, s6, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB26_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw add ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_add_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_add_i64_ret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB27_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB27_3 |
| ; GCN1-NEXT: s_branch .LBB27_4 |
| ; GCN1-NEXT: .LBB27_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB27_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, s6, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB27_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_add_i64_ret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB27_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB27_3 |
| ; GCN2-NEXT: s_branch .LBB27_4 |
| ; GCN2-NEXT: .LBB27_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB27_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, s6, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v4, vcc, v1, v4, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB27_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_add_i64_ret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB27_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB27_3 |
| ; GCN3-NEXT: s_branch .LBB27_4 |
| ; GCN3-NEXT: .LBB27_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB27_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, s6, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB27_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw add ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define void @flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB28_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB28_4 |
| ; GCN1-NEXT: .LBB28_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB28_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB28_2 |
| ; GCN1-NEXT: .LBB28_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, v1, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB28_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB28_4 |
| ; GCN2-NEXT: .LBB28_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB28_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB28_2 |
| ; GCN2-NEXT: .LBB28_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, v1, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v2, vcc, v5, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB28_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB28_4 |
| ; GCN3-NEXT: .LBB28_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB28_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB28_2 |
| ; GCN3-NEXT: .LBB28_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v1, vcc, v1, v2 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v2, vcc, v4, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret void |
| } |
| |
| define i64 @flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB29_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB29_4 |
| ; GCN1-NEXT: .LBB29_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB29_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB29_2 |
| ; GCN1-NEXT: .LBB29_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v2, vcc, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB29_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB29_4 |
| ; GCN2-NEXT: .LBB29_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB29_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB29_2 |
| ; GCN2-NEXT: .LBB29_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v2, vcc, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB29_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB29_4 |
| ; GCN3-NEXT: .LBB29_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB29_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB29_2 |
| ; GCN3-NEXT: .LBB29_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw add ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret i64 %result |
| } |
| |
| ; --------------------------------------------------------------------- |
| ; atomicrmw sub |
| ; --------------------------------------------------------------------- |
| |
| define void @flat_atomic_sub_i64_noret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_sub_i64_noret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB30_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB30_4 |
| ; GCN1-NEXT: .LBB30_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB30_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB30_2 |
| ; GCN1-NEXT: .LBB30_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_sub_i64_noret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB30_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB30_4 |
| ; GCN2-NEXT: .LBB30_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB30_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB30_2 |
| ; GCN2-NEXT: .LBB30_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_sub_i64_noret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB30_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB30_4 |
| ; GCN3-NEXT: .LBB30_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB30_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB30_2 |
| ; GCN3-NEXT: .LBB30_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define void @flat_atomic_sub_i64_noret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_sub_i64_noret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB31_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB31_4 |
| ; GCN1-NEXT: .LBB31_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB31_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB31_2 |
| ; GCN1-NEXT: .LBB31_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_sub_i64_noret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB31_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB31_4 |
| ; GCN2-NEXT: .LBB31_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB31_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB31_2 |
| ; GCN2-NEXT: .LBB31_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_sub_i64_noret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB31_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB31_4 |
| ; GCN3-NEXT: .LBB31_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB31_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB31_2 |
| ; GCN3-NEXT: .LBB31_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define i64 @flat_atomic_sub_i64_ret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_sub_i64_ret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB32_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB32_4 |
| ; GCN1-NEXT: .LBB32_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB32_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB32_2 |
| ; GCN1-NEXT: .LBB32_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_sub_i64_ret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB32_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB32_4 |
| ; GCN2-NEXT: .LBB32_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB32_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB32_2 |
| ; GCN2-NEXT: .LBB32_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_sub_i64_ret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB32_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB32_4 |
| ; GCN3-NEXT: .LBB32_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB32_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB32_2 |
| ; GCN3-NEXT: .LBB32_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw sub ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define i64 @flat_atomic_sub_i64_ret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_sub_i64_ret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB33_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB33_4 |
| ; GCN1-NEXT: .LBB33_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB33_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB33_2 |
| ; GCN1-NEXT: .LBB33_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_sub_i64_ret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB33_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB33_4 |
| ; GCN2-NEXT: .LBB33_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB33_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB33_2 |
| ; GCN2-NEXT: .LBB33_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_sub_i64_ret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB33_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB33_4 |
| ; GCN3-NEXT: .LBB33_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB33_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB33_2 |
| ; GCN3-NEXT: .LBB33_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw sub ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_sub_i64_noret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB34_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB34_4 |
| ; GCN1-NEXT: .LBB34_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB34_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB34_2 |
| ; GCN1-NEXT: .LBB34_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_subrev_i32_e32 v1, vcc, s6, v1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_sub_i64_noret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB34_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB34_4 |
| ; GCN2-NEXT: .LBB34_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB34_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB34_2 |
| ; GCN2-NEXT: .LBB34_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_subrev_u32_e32 v1, vcc, s6, v1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_sub_i64_noret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB34_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB34_4 |
| ; GCN3-NEXT: .LBB34_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB34_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB34_2 |
| ; GCN3-NEXT: .LBB34_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_subrev_co_u32_e32 v1, vcc, s6, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_sub_i64_noret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB35_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB35_4 |
| ; GCN1-NEXT: .LBB35_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB35_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB35_2 |
| ; GCN1-NEXT: .LBB35_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_subrev_i32_e32 v1, vcc, s6, v1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_sub_i64_noret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB35_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB35_4 |
| ; GCN2-NEXT: .LBB35_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB35_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB35_2 |
| ; GCN2-NEXT: .LBB35_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_subrev_u32_e32 v1, vcc, s6, v1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v3, v4, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_sub_i64_noret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB35_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB35_4 |
| ; GCN3-NEXT: .LBB35_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB35_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB35_2 |
| ; GCN3-NEXT: .LBB35_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_subrev_co_u32_e32 v1, vcc, s6, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_sub_i64_ret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB36_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB36_3 |
| ; GCN1-NEXT: s_branch .LBB36_4 |
| ; GCN1-NEXT: .LBB36_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB36_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB36_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_sub_i64_ret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB36_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB36_3 |
| ; GCN2-NEXT: s_branch .LBB36_4 |
| ; GCN2-NEXT: .LBB36_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB36_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB36_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_sub_i64_ret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB36_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB36_3 |
| ; GCN3-NEXT: s_branch .LBB36_4 |
| ; GCN3-NEXT: .LBB36_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB36_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB36_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw sub ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_sub_i64_ret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB37_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB37_3 |
| ; GCN1-NEXT: s_branch .LBB37_4 |
| ; GCN1-NEXT: .LBB37_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB37_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB37_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_sub_i64_ret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB37_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB37_3 |
| ; GCN2-NEXT: s_branch .LBB37_4 |
| ; GCN2-NEXT: .LBB37_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB37_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_subb_u32_e32 v4, vcc, v1, v4, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB37_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_sub_i64_ret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB37_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB37_3 |
| ; GCN3-NEXT: s_branch .LBB37_4 |
| ; GCN3-NEXT: .LBB37_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB37_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB37_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw sub ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define void @flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB38_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB38_4 |
| ; GCN1-NEXT: .LBB38_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB38_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB38_2 |
| ; GCN1-NEXT: .LBB38_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_sub_i32_e32 v1, vcc, v1, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB38_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB38_4 |
| ; GCN2-NEXT: .LBB38_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB38_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB38_2 |
| ; GCN2-NEXT: .LBB38_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v5, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_sub_u32_e32 v1, vcc, v1, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_subb_u32_e32 v2, vcc, v5, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB38_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB38_4 |
| ; GCN3-NEXT: .LBB38_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB38_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB38_2 |
| ; GCN3-NEXT: .LBB38_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v2 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_subb_co_u32_e32 v2, vcc, v4, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret void |
| } |
| |
| define i64 @flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB39_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB39_4 |
| ; GCN1-NEXT: .LBB39_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB39_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB39_2 |
| ; GCN1-NEXT: .LBB39_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_sub_i32_e32 v2, vcc, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB39_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB39_4 |
| ; GCN2-NEXT: .LBB39_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB39_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB39_2 |
| ; GCN2-NEXT: .LBB39_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_sub_u32_e32 v2, vcc, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_subb_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB39_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB39_4 |
| ; GCN3-NEXT: .LBB39_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB39_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB39_2 |
| ; GCN3-NEXT: .LBB39_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v2 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw sub ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret i64 %result |
| } |
| |
| ; --------------------------------------------------------------------- |
| ; atomicrmw and |
| ; --------------------------------------------------------------------- |
| |
| define void @flat_atomic_and_i64_noret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_and_i64_noret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB40_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB40_4 |
| ; GCN1-NEXT: .LBB40_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB40_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB40_2 |
| ; GCN1-NEXT: .LBB40_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_and_i64_noret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB40_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB40_4 |
| ; GCN2-NEXT: .LBB40_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB40_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB40_2 |
| ; GCN2-NEXT: .LBB40_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_and_i64_noret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB40_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB40_4 |
| ; GCN3-NEXT: .LBB40_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB40_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB40_2 |
| ; GCN3-NEXT: .LBB40_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v1, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define void @flat_atomic_and_i64_noret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_and_i64_noret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB41_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB41_4 |
| ; GCN1-NEXT: .LBB41_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB41_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB41_2 |
| ; GCN1-NEXT: .LBB41_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_and_i64_noret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB41_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB41_4 |
| ; GCN2-NEXT: .LBB41_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB41_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB41_2 |
| ; GCN2-NEXT: .LBB41_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_and_i64_noret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB41_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB41_4 |
| ; GCN3-NEXT: .LBB41_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB41_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB41_2 |
| ; GCN3-NEXT: .LBB41_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v1, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define i64 @flat_atomic_and_i64_ret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_and_i64_ret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB42_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB42_4 |
| ; GCN1-NEXT: .LBB42_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB42_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB42_2 |
| ; GCN1-NEXT: .LBB42_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_and_i64_ret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB42_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB42_4 |
| ; GCN2-NEXT: .LBB42_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB42_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB42_2 |
| ; GCN2-NEXT: .LBB42_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_and_i64_ret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB42_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB42_4 |
| ; GCN3-NEXT: .LBB42_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB42_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB42_2 |
| ; GCN3-NEXT: .LBB42_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw and ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define i64 @flat_atomic_and_i64_ret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_and_i64_ret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB43_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB43_4 |
| ; GCN1-NEXT: .LBB43_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB43_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB43_2 |
| ; GCN1-NEXT: .LBB43_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_and_i64_ret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB43_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB43_4 |
| ; GCN2-NEXT: .LBB43_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB43_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB43_2 |
| ; GCN2-NEXT: .LBB43_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_and_i64_ret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB43_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB43_4 |
| ; GCN3-NEXT: .LBB43_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB43_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB43_2 |
| ; GCN3-NEXT: .LBB43_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw and ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_and_i64_noret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB44_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB44_4 |
| ; GCN1-NEXT: .LBB44_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB44_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB44_2 |
| ; GCN1-NEXT: .LBB44_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, s6, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, s7, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_and_i64_noret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB44_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB44_4 |
| ; GCN2-NEXT: .LBB44_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB44_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB44_2 |
| ; GCN2-NEXT: .LBB44_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, s6, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, s7, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_and_i64_noret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB44_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB44_4 |
| ; GCN3-NEXT: .LBB44_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB44_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB44_2 |
| ; GCN3-NEXT: .LBB44_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v1, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, s6, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_and_i64_noret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB45_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB45_4 |
| ; GCN1-NEXT: .LBB45_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB45_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB45_2 |
| ; GCN1-NEXT: .LBB45_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, s6, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, s7, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_and_i64_noret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB45_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB45_4 |
| ; GCN2-NEXT: .LBB45_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB45_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB45_2 |
| ; GCN2-NEXT: .LBB45_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, s6, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, s7, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_and_i64_noret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB45_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB45_4 |
| ; GCN3-NEXT: .LBB45_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB45_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB45_2 |
| ; GCN3-NEXT: .LBB45_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v1, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, s6, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_and_i64_ret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB46_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB46_3 |
| ; GCN1-NEXT: s_branch .LBB46_4 |
| ; GCN1-NEXT: .LBB46_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB46_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v4, s6, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v5, s7, v1 |
| ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB46_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_and_i64_ret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB46_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB46_3 |
| ; GCN2-NEXT: s_branch .LBB46_4 |
| ; GCN2-NEXT: .LBB46_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB46_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v4, s6, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v5, s7, v1 |
| ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB46_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_and_i64_ret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB46_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB46_3 |
| ; GCN3-NEXT: s_branch .LBB46_4 |
| ; GCN3-NEXT: .LBB46_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB46_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v3, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v4, s6, v0 |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB46_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw and ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_and_i64_ret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB47_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB47_3 |
| ; GCN1-NEXT: s_branch .LBB47_4 |
| ; GCN1-NEXT: .LBB47_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB47_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v4, s6, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v5, s7, v1 |
| ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB47_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_and_i64_ret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB47_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB47_3 |
| ; GCN2-NEXT: s_branch .LBB47_4 |
| ; GCN2-NEXT: .LBB47_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB47_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v4, s6, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v5, s7, v1 |
| ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB47_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_and_i64_ret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB47_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB47_3 |
| ; GCN3-NEXT: s_branch .LBB47_4 |
| ; GCN3-NEXT: .LBB47_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB47_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v3, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v4, s6, v0 |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB47_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw and ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define void @flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB48_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB48_4 |
| ; GCN1-NEXT: .LBB48_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB48_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB48_2 |
| ; GCN1-NEXT: .LBB48_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB48_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB48_4 |
| ; GCN2-NEXT: .LBB48_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB48_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB48_2 |
| ; GCN2-NEXT: .LBB48_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB48_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB48_4 |
| ; GCN3-NEXT: .LBB48_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB48_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB48_2 |
| ; GCN3-NEXT: .LBB48_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v1, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret void |
| } |
| |
| define i64 @flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB49_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB49_4 |
| ; GCN1-NEXT: .LBB49_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB49_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB49_2 |
| ; GCN1-NEXT: .LBB49_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB49_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB49_4 |
| ; GCN2-NEXT: .LBB49_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB49_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB49_2 |
| ; GCN2-NEXT: .LBB49_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB49_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB49_4 |
| ; GCN3-NEXT: .LBB49_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB49_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB49_2 |
| ; GCN3-NEXT: .LBB49_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw and ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret i64 %result |
| } |
| |
| ; --------------------------------------------------------------------- |
| ; atomicrmw nand |
| ; --------------------------------------------------------------------- |
| |
| define void @flat_atomic_nand_i64_noret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_nand_i64_noret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB50_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB50_6 |
| ; GCN1-NEXT: .LBB50_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB50_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB50_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v4, v7, v3 |
| ; GCN1-NEXT: v_and_b32_e32 v8, v6, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v5, v4 |
| ; GCN1-NEXT: v_not_b32_e32 v4, v8 |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB50_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB50_2 |
| ; GCN1-NEXT: .LBB50_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 |
| ; GCN1-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_nand_i64_noret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB50_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB50_6 |
| ; GCN2-NEXT: .LBB50_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB50_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB50_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v4, v7, v3 |
| ; GCN2-NEXT: v_and_b32_e32 v8, v6, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v5, v4 |
| ; GCN2-NEXT: v_not_b32_e32 v4, v8 |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB50_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB50_2 |
| ; GCN2-NEXT: .LBB50_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 |
| ; GCN2-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_nand_i64_noret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB50_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB50_6 |
| ; GCN3-NEXT: .LBB50_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB50_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB50_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v4, v7, v3 |
| ; GCN3-NEXT: v_and_b32_e32 v8, v6, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v5, v4 |
| ; GCN3-NEXT: v_not_b32_e32 v4, v8 |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB50_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB50_2 |
| ; GCN3-NEXT: .LBB50_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v1, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v1, v1 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw nand ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define void @flat_atomic_nand_i64_noret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_nand_i64_noret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB51_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB51_6 |
| ; GCN1-NEXT: .LBB51_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB51_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB51_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v4, v7, v3 |
| ; GCN1-NEXT: v_and_b32_e32 v8, v6, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v5, v4 |
| ; GCN1-NEXT: v_not_b32_e32 v4, v8 |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB51_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB51_2 |
| ; GCN1-NEXT: .LBB51_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 |
| ; GCN1-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_nand_i64_noret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB51_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB51_6 |
| ; GCN2-NEXT: .LBB51_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB51_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB51_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v4, v7, v3 |
| ; GCN2-NEXT: v_and_b32_e32 v8, v6, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v5, v4 |
| ; GCN2-NEXT: v_not_b32_e32 v4, v8 |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB51_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB51_2 |
| ; GCN2-NEXT: .LBB51_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 |
| ; GCN2-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_nand_i64_noret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB51_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB51_6 |
| ; GCN3-NEXT: .LBB51_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB51_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB51_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v4, v7, v3 |
| ; GCN3-NEXT: v_and_b32_e32 v8, v6, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v5, v4 |
| ; GCN3-NEXT: v_not_b32_e32 v4, v8 |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB51_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB51_2 |
| ; GCN3-NEXT: .LBB51_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v1, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v1, v1 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define i64 @flat_atomic_nand_i64_ret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_nand_i64_ret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB52_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v5, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v4, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB52_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: v_and_b32_e32 v4, v7, v3 |
| ; GCN1-NEXT: v_and_b32_e32 v8, v6, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v5, v4 |
| ; GCN1-NEXT: v_not_b32_e32 v4, v8 |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB52_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: .LBB52_4: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB52_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 |
| ; GCN1-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB52_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: v_mov_b32_e32 v0, v4 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, v5 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_nand_i64_ret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB52_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v5, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v4, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB52_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: v_and_b32_e32 v4, v7, v3 |
| ; GCN2-NEXT: v_and_b32_e32 v8, v6, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v5, v4 |
| ; GCN2-NEXT: v_not_b32_e32 v4, v8 |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB52_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: .LBB52_4: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB52_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 |
| ; GCN2-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB52_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: v_mov_b32_e32 v0, v4 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, v5 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_nand_i64_ret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB52_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB52_6 |
| ; GCN3-NEXT: .LBB52_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB52_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB52_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_and_b32_e32 v0, v9, v3 |
| ; GCN3-NEXT: v_and_b32_e32 v1, v8, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v7, v0 |
| ; GCN3-NEXT: v_not_b32_e32 v6, v1 |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB52_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB52_2 |
| ; GCN3-NEXT: .LBB52_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw nand ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define i64 @flat_atomic_nand_i64_ret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_nand_i64_ret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB53_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB53_6 |
| ; GCN1-NEXT: .LBB53_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB53_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB53_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_and_b32_e32 v0, v9, v3 |
| ; GCN1-NEXT: v_and_b32_e32 v1, v8, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v7, v0 |
| ; GCN1-NEXT: v_not_b32_e32 v6, v1 |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB53_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB53_2 |
| ; GCN1-NEXT: .LBB53_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN1-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_nand_i64_ret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB53_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB53_6 |
| ; GCN2-NEXT: .LBB53_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB53_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB53_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_and_b32_e32 v0, v9, v3 |
| ; GCN2-NEXT: v_and_b32_e32 v1, v8, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v7, v0 |
| ; GCN2-NEXT: v_not_b32_e32 v6, v1 |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB53_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB53_2 |
| ; GCN2-NEXT: .LBB53_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN2-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_nand_i64_ret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB53_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB53_6 |
| ; GCN3-NEXT: .LBB53_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB53_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB53_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_and_b32_e32 v0, v9, v3 |
| ; GCN3-NEXT: v_and_b32_e32 v1, v8, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v7, v0 |
| ; GCN3-NEXT: v_not_b32_e32 v6, v1 |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB53_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB53_2 |
| ; GCN3-NEXT: .LBB53_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw nand ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_nand_i64_noret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB54_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB54_6 |
| ; GCN1-NEXT: .LBB54_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB54_3: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN1-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN1-NEXT: .LBB54_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v0, s7, v3 |
| ; GCN1-NEXT: v_and_b32_e32 v6, s6, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v1, v0 |
| ; GCN1-NEXT: v_not_b32_e32 v0, v6 |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB54_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_branch .LBB54_2 |
| ; GCN1-NEXT: .LBB54_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, s6, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, s7, v3 |
| ; GCN1-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_nand_i64_noret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB54_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB54_6 |
| ; GCN2-NEXT: .LBB54_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB54_3: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN2-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN2-NEXT: .LBB54_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v0, s7, v3 |
| ; GCN2-NEXT: v_and_b32_e32 v6, s6, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v1, v0 |
| ; GCN2-NEXT: v_not_b32_e32 v0, v6 |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB54_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_branch .LBB54_2 |
| ; GCN2-NEXT: .LBB54_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, s6, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, s7, v3 |
| ; GCN2-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_nand_i64_noret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB54_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB54_6 |
| ; GCN3-NEXT: .LBB54_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB54_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN3-NEXT: .LBB54_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v0, s7, v3 |
| ; GCN3-NEXT: v_and_b32_e32 v6, s6, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v1, v0 |
| ; GCN3-NEXT: v_not_b32_e32 v0, v6 |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB54_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_branch .LBB54_2 |
| ; GCN3-NEXT: .LBB54_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v1, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, s6, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v1, v1 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw nand ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_nand_i64_noret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB55_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB55_6 |
| ; GCN1-NEXT: .LBB55_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB55_3: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN1-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN1-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN1-NEXT: .LBB55_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v0, s7, v3 |
| ; GCN1-NEXT: v_and_b32_e32 v6, s6, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v1, v0 |
| ; GCN1-NEXT: v_not_b32_e32 v0, v6 |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB55_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_branch .LBB55_2 |
| ; GCN1-NEXT: .LBB55_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, s6, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, s7, v3 |
| ; GCN1-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_nand_i64_noret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB55_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB55_6 |
| ; GCN2-NEXT: .LBB55_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB55_3: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN2-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN2-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN2-NEXT: .LBB55_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v0, s7, v3 |
| ; GCN2-NEXT: v_and_b32_e32 v6, s6, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v1, v0 |
| ; GCN2-NEXT: v_not_b32_e32 v0, v6 |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB55_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_branch .LBB55_2 |
| ; GCN2-NEXT: .LBB55_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, s6, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, s7, v3 |
| ; GCN2-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_nand_i64_noret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB55_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB55_6 |
| ; GCN3-NEXT: .LBB55_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB55_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN3-NEXT: .LBB55_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v0, s7, v3 |
| ; GCN3-NEXT: v_and_b32_e32 v6, s6, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v1, v0 |
| ; GCN3-NEXT: v_not_b32_e32 v0, v6 |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB55_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_branch .LBB55_2 |
| ; GCN3-NEXT: .LBB55_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v1, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, s6, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v1, v1 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_nand_i64_ret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB56_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN1-NEXT: .LBB56_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v0 |
| ; GCN1-NEXT: v_and_b32_e32 v0, s7, v7 |
| ; GCN1-NEXT: v_and_b32_e32 v1, s6, v6 |
| ; GCN1-NEXT: v_not_b32_e32 v5, v0 |
| ; GCN1-NEXT: v_not_b32_e32 v4, v1 |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] |
| ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB56_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_branch .LBB56_6 |
| ; GCN1-NEXT: .LBB56_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB56_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v4, s6, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v5, s7, v1 |
| ; GCN1-NEXT: v_not_b32_e32 v4, v4 |
| ; GCN1-NEXT: v_not_b32_e32 v5, v5 |
| ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB56_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_nand_i64_ret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB56_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN2-NEXT: .LBB56_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v0 |
| ; GCN2-NEXT: v_and_b32_e32 v0, s7, v7 |
| ; GCN2-NEXT: v_and_b32_e32 v1, s6, v6 |
| ; GCN2-NEXT: v_not_b32_e32 v5, v0 |
| ; GCN2-NEXT: v_not_b32_e32 v4, v1 |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] |
| ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB56_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_branch .LBB56_6 |
| ; GCN2-NEXT: .LBB56_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB56_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v4, s6, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v5, s7, v1 |
| ; GCN2-NEXT: v_not_b32_e32 v4, v4 |
| ; GCN2-NEXT: v_not_b32_e32 v5, v5 |
| ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB56_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_nand_i64_ret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB56_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN3-NEXT: .LBB56_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v0 |
| ; GCN3-NEXT: v_and_b32_e32 v0, s7, v7 |
| ; GCN3-NEXT: v_and_b32_e32 v1, s6, v6 |
| ; GCN3-NEXT: v_not_b32_e32 v5, v0 |
| ; GCN3-NEXT: v_not_b32_e32 v4, v1 |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] |
| ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB56_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_branch .LBB56_6 |
| ; GCN3-NEXT: .LBB56_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB56_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v3, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v4, s6, v0 |
| ; GCN3-NEXT: v_not_b32_e32 v4, v4 |
| ; GCN3-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB56_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw nand ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_nand_i64_ret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB57_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN1-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN1-NEXT: .LBB57_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v0 |
| ; GCN1-NEXT: v_and_b32_e32 v0, s7, v7 |
| ; GCN1-NEXT: v_and_b32_e32 v1, s6, v6 |
| ; GCN1-NEXT: v_not_b32_e32 v5, v0 |
| ; GCN1-NEXT: v_not_b32_e32 v4, v1 |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] |
| ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB57_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_branch .LBB57_6 |
| ; GCN1-NEXT: .LBB57_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB57_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v4, s6, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v5, s7, v1 |
| ; GCN1-NEXT: v_not_b32_e32 v4, v4 |
| ; GCN1-NEXT: v_not_b32_e32 v5, v5 |
| ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB57_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_nand_i64_ret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB57_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN2-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN2-NEXT: .LBB57_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v0 |
| ; GCN2-NEXT: v_and_b32_e32 v0, s7, v7 |
| ; GCN2-NEXT: v_and_b32_e32 v1, s6, v6 |
| ; GCN2-NEXT: v_not_b32_e32 v5, v0 |
| ; GCN2-NEXT: v_not_b32_e32 v4, v1 |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] |
| ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB57_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_branch .LBB57_6 |
| ; GCN2-NEXT: .LBB57_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB57_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v4, s6, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v5, s7, v1 |
| ; GCN2-NEXT: v_not_b32_e32 v4, v4 |
| ; GCN2-NEXT: v_not_b32_e32 v5, v5 |
| ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB57_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_nand_i64_ret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB57_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN3-NEXT: .LBB57_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v0 |
| ; GCN3-NEXT: v_and_b32_e32 v0, s7, v7 |
| ; GCN3-NEXT: v_and_b32_e32 v1, s6, v6 |
| ; GCN3-NEXT: v_not_b32_e32 v5, v0 |
| ; GCN3-NEXT: v_not_b32_e32 v4, v1 |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7] |
| ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB57_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_branch .LBB57_6 |
| ; GCN3-NEXT: .LBB57_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB57_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v3, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v4, s6, v0 |
| ; GCN3-NEXT: v_not_b32_e32 v4, v4 |
| ; GCN3-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB57_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw nand ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define void @flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB58_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB58_6 |
| ; GCN1-NEXT: .LBB58_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB58_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB58_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v4, v7, v3 |
| ; GCN1-NEXT: v_and_b32_e32 v8, v6, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v5, v4 |
| ; GCN1-NEXT: v_not_b32_e32 v4, v8 |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB58_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB58_2 |
| ; GCN1-NEXT: .LBB58_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, v5, v3 |
| ; GCN1-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB58_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB58_6 |
| ; GCN2-NEXT: .LBB58_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB58_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB58_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v4, v7, v3 |
| ; GCN2-NEXT: v_and_b32_e32 v8, v6, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v5, v4 |
| ; GCN2-NEXT: v_not_b32_e32 v4, v8 |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB58_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB58_2 |
| ; GCN2-NEXT: .LBB58_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, v5, v3 |
| ; GCN2-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB58_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB58_6 |
| ; GCN3-NEXT: .LBB58_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB58_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB58_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v4, v7, v3 |
| ; GCN3-NEXT: v_and_b32_e32 v8, v6, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v5, v4 |
| ; GCN3-NEXT: v_not_b32_e32 v4, v8 |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB58_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB58_2 |
| ; GCN3-NEXT: .LBB58_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v1, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, v4, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v1, v1 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret void |
| } |
| |
| define i64 @flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB59_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB59_6 |
| ; GCN1-NEXT: .LBB59_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB59_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB59_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_and_b32_e32 v0, v9, v3 |
| ; GCN1-NEXT: v_and_b32_e32 v1, v8, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v7, v0 |
| ; GCN1-NEXT: v_not_b32_e32 v6, v1 |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB59_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB59_2 |
| ; GCN1-NEXT: .LBB59_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN1-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN1-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB59_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB59_6 |
| ; GCN2-NEXT: .LBB59_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB59_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB59_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_and_b32_e32 v0, v9, v3 |
| ; GCN2-NEXT: v_and_b32_e32 v1, v8, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v7, v0 |
| ; GCN2-NEXT: v_not_b32_e32 v6, v1 |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB59_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB59_2 |
| ; GCN2-NEXT: .LBB59_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN2-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN2-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB59_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB59_6 |
| ; GCN3-NEXT: .LBB59_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB59_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB59_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_and_b32_e32 v0, v9, v3 |
| ; GCN3-NEXT: v_and_b32_e32 v1, v8, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v7, v0 |
| ; GCN3-NEXT: v_not_b32_e32 v6, v1 |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB59_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB59_2 |
| ; GCN3-NEXT: .LBB59_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_and_b32_e32 v3, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_and_b32_e32 v2, v0, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v2, v2 |
| ; GCN3-NEXT: v_not_b32_e32 v3, v3 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw nand ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret i64 %result |
| } |
| |
| ; --------------------------------------------------------------------- |
| ; atomicrmw or |
| ; --------------------------------------------------------------------- |
| |
| define void @flat_atomic_or_i64_noret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_or_i64_noret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB60_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB60_4 |
| ; GCN1-NEXT: .LBB60_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB60_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB60_2 |
| ; GCN1-NEXT: .LBB60_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_or_b32_e32 v2, v4, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_or_b32_e32 v3, v5, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_or_i64_noret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB60_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB60_4 |
| ; GCN2-NEXT: .LBB60_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB60_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB60_2 |
| ; GCN2-NEXT: .LBB60_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_or_b32_e32 v2, v4, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_or_b32_e32 v3, v5, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_or_i64_noret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB60_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB60_4 |
| ; GCN3-NEXT: .LBB60_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB60_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB60_2 |
| ; GCN3-NEXT: .LBB60_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_or_b32_e32 v1, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_or_b32_e32 v2, v4, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define void @flat_atomic_or_i64_noret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_or_i64_noret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB61_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB61_4 |
| ; GCN1-NEXT: .LBB61_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB61_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB61_2 |
| ; GCN1-NEXT: .LBB61_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_or_b32_e32 v2, v4, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_or_b32_e32 v3, v5, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_or_i64_noret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB61_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB61_4 |
| ; GCN2-NEXT: .LBB61_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB61_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB61_2 |
| ; GCN2-NEXT: .LBB61_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_or_b32_e32 v2, v4, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_or_b32_e32 v3, v5, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_or_i64_noret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB61_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB61_4 |
| ; GCN3-NEXT: .LBB61_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB61_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB61_2 |
| ; GCN3-NEXT: .LBB61_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_or_b32_e32 v1, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_or_b32_e32 v2, v4, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define i64 @flat_atomic_or_i64_ret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_or_i64_ret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB62_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB62_4 |
| ; GCN1-NEXT: .LBB62_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB62_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB62_2 |
| ; GCN1-NEXT: .LBB62_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_or_b32_e32 v2, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_or_b32_e32 v3, v1, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_or_i64_ret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB62_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB62_4 |
| ; GCN2-NEXT: .LBB62_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB62_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB62_2 |
| ; GCN2-NEXT: .LBB62_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_or_b32_e32 v2, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_or_b32_e32 v3, v1, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_or_i64_ret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB62_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB62_4 |
| ; GCN3-NEXT: .LBB62_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB62_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB62_2 |
| ; GCN3-NEXT: .LBB62_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_or_b32_e32 v3, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_or_b32_e32 v2, v0, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw or ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define i64 @flat_atomic_or_i64_ret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_or_i64_ret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB63_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB63_4 |
| ; GCN1-NEXT: .LBB63_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB63_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB63_2 |
| ; GCN1-NEXT: .LBB63_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_or_b32_e32 v2, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_or_b32_e32 v3, v1, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_or_i64_ret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB63_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB63_4 |
| ; GCN2-NEXT: .LBB63_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB63_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB63_2 |
| ; GCN2-NEXT: .LBB63_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_or_b32_e32 v2, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_or_b32_e32 v3, v1, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_or_i64_ret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB63_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB63_4 |
| ; GCN3-NEXT: .LBB63_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB63_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB63_2 |
| ; GCN3-NEXT: .LBB63_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_or_b32_e32 v3, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_or_b32_e32 v2, v0, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw or ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_or_i64_noret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB64_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB64_4 |
| ; GCN1-NEXT: .LBB64_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB64_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB64_2 |
| ; GCN1-NEXT: .LBB64_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_or_b32_e32 v2, s6, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_or_b32_e32 v3, s7, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_or_i64_noret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB64_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB64_4 |
| ; GCN2-NEXT: .LBB64_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB64_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB64_2 |
| ; GCN2-NEXT: .LBB64_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_or_b32_e32 v2, s6, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_or_b32_e32 v3, s7, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_or_i64_noret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB64_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB64_4 |
| ; GCN3-NEXT: .LBB64_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB64_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB64_2 |
| ; GCN3-NEXT: .LBB64_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_or_b32_e32 v1, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_or_b32_e32 v2, s6, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_or_i64_noret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB65_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB65_4 |
| ; GCN1-NEXT: .LBB65_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB65_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB65_2 |
| ; GCN1-NEXT: .LBB65_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_or_b32_e32 v2, s6, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_or_b32_e32 v3, s7, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_or_i64_noret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB65_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB65_4 |
| ; GCN2-NEXT: .LBB65_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB65_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB65_2 |
| ; GCN2-NEXT: .LBB65_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_or_b32_e32 v2, s6, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_or_b32_e32 v3, s7, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_or_i64_noret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB65_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB65_4 |
| ; GCN3-NEXT: .LBB65_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB65_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB65_2 |
| ; GCN3-NEXT: .LBB65_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_or_b32_e32 v1, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_or_b32_e32 v2, s6, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_or_i64_ret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB66_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB66_3 |
| ; GCN1-NEXT: s_branch .LBB66_4 |
| ; GCN1-NEXT: .LBB66_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB66_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_or_b32_e32 v4, s6, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_or_b32_e32 v5, s7, v1 |
| ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB66_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_or_i64_ret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB66_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB66_3 |
| ; GCN2-NEXT: s_branch .LBB66_4 |
| ; GCN2-NEXT: .LBB66_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB66_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_or_b32_e32 v4, s6, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_or_b32_e32 v5, s7, v1 |
| ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB66_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_or_i64_ret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB66_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB66_3 |
| ; GCN3-NEXT: s_branch .LBB66_4 |
| ; GCN3-NEXT: .LBB66_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB66_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_or_b32_e32 v3, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_or_b32_e32 v4, s6, v0 |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB66_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw or ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_or_i64_ret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB67_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB67_3 |
| ; GCN1-NEXT: s_branch .LBB67_4 |
| ; GCN1-NEXT: .LBB67_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB67_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_or_b32_e32 v4, s6, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_or_b32_e32 v5, s7, v1 |
| ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB67_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_or_i64_ret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB67_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB67_3 |
| ; GCN2-NEXT: s_branch .LBB67_4 |
| ; GCN2-NEXT: .LBB67_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB67_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_or_b32_e32 v4, s6, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_or_b32_e32 v5, s7, v1 |
| ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB67_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_or_i64_ret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB67_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB67_3 |
| ; GCN3-NEXT: s_branch .LBB67_4 |
| ; GCN3-NEXT: .LBB67_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB67_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_or_b32_e32 v3, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_or_b32_e32 v4, s6, v0 |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB67_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw or ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define void @flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB68_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB68_4 |
| ; GCN1-NEXT: .LBB68_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB68_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB68_2 |
| ; GCN1-NEXT: .LBB68_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_or_b32_e32 v2, v4, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_or_b32_e32 v3, v5, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB68_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB68_4 |
| ; GCN2-NEXT: .LBB68_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB68_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB68_2 |
| ; GCN2-NEXT: .LBB68_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_or_b32_e32 v2, v4, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_or_b32_e32 v3, v5, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB68_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB68_4 |
| ; GCN3-NEXT: .LBB68_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB68_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB68_2 |
| ; GCN3-NEXT: .LBB68_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_or_b32_e32 v1, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_or_b32_e32 v2, v4, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret void |
| } |
| |
| define i64 @flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB69_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB69_4 |
| ; GCN1-NEXT: .LBB69_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB69_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB69_2 |
| ; GCN1-NEXT: .LBB69_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_or_b32_e32 v2, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_or_b32_e32 v3, v1, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB69_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB69_4 |
| ; GCN2-NEXT: .LBB69_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB69_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB69_2 |
| ; GCN2-NEXT: .LBB69_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_or_b32_e32 v2, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_or_b32_e32 v3, v1, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB69_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB69_4 |
| ; GCN3-NEXT: .LBB69_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB69_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB69_2 |
| ; GCN3-NEXT: .LBB69_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_or_b32_e32 v3, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_or_b32_e32 v2, v0, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw or ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret i64 %result |
| } |
| |
| ; --------------------------------------------------------------------- |
| ; atomicrmw xor |
| ; --------------------------------------------------------------------- |
| |
| define void @flat_atomic_xor_i64_noret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_xor_i64_noret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB70_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB70_4 |
| ; GCN1-NEXT: .LBB70_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB70_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB70_2 |
| ; GCN1-NEXT: .LBB70_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xor_i64_noret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB70_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB70_4 |
| ; GCN2-NEXT: .LBB70_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB70_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB70_2 |
| ; GCN2-NEXT: .LBB70_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xor_i64_noret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB70_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB70_4 |
| ; GCN3-NEXT: .LBB70_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB70_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB70_2 |
| ; GCN3-NEXT: .LBB70_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define void @flat_atomic_xor_i64_noret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_xor_i64_noret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB71_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB71_4 |
| ; GCN1-NEXT: .LBB71_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB71_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB71_2 |
| ; GCN1-NEXT: .LBB71_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xor_i64_noret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB71_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB71_4 |
| ; GCN2-NEXT: .LBB71_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB71_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB71_2 |
| ; GCN2-NEXT: .LBB71_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xor_i64_noret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB71_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB71_4 |
| ; GCN3-NEXT: .LBB71_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB71_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB71_2 |
| ; GCN3-NEXT: .LBB71_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define i64 @flat_atomic_xor_i64_ret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_xor_i64_ret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB72_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB72_4 |
| ; GCN1-NEXT: .LBB72_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB72_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB72_2 |
| ; GCN1-NEXT: .LBB72_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_xor_b32_e32 v2, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_xor_b32_e32 v3, v1, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xor_i64_ret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB72_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB72_4 |
| ; GCN2-NEXT: .LBB72_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB72_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB72_2 |
| ; GCN2-NEXT: .LBB72_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_xor_b32_e32 v2, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_xor_b32_e32 v3, v1, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xor_i64_ret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB72_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB72_4 |
| ; GCN3-NEXT: .LBB72_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB72_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB72_2 |
| ; GCN3-NEXT: .LBB72_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw xor ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define i64 @flat_atomic_xor_i64_ret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_xor_i64_ret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB73_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB73_4 |
| ; GCN1-NEXT: .LBB73_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB73_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB73_2 |
| ; GCN1-NEXT: .LBB73_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_xor_b32_e32 v2, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_xor_b32_e32 v3, v1, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xor_i64_ret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB73_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB73_4 |
| ; GCN2-NEXT: .LBB73_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB73_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB73_2 |
| ; GCN2-NEXT: .LBB73_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_xor_b32_e32 v2, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_xor_b32_e32 v3, v1, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xor_i64_ret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB73_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB73_4 |
| ; GCN3-NEXT: .LBB73_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB73_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB73_2 |
| ; GCN3-NEXT: .LBB73_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw xor ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_xor_i64_noret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB74_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB74_4 |
| ; GCN1-NEXT: .LBB74_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB74_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB74_2 |
| ; GCN1-NEXT: .LBB74_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_xor_b32_e32 v2, s6, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_xor_b32_e32 v3, s7, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xor_i64_noret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB74_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB74_4 |
| ; GCN2-NEXT: .LBB74_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB74_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB74_2 |
| ; GCN2-NEXT: .LBB74_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_xor_b32_e32 v2, s6, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_xor_b32_e32 v3, s7, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xor_i64_noret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB74_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB74_4 |
| ; GCN3-NEXT: .LBB74_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB74_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB74_2 |
| ; GCN3-NEXT: .LBB74_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_xor_b32_e32 v1, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_xor_b32_e32 v2, s6, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_xor_i64_noret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB75_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB75_4 |
| ; GCN1-NEXT: .LBB75_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB75_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB75_2 |
| ; GCN1-NEXT: .LBB75_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN1-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_xor_b32_e32 v2, s6, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_xor_b32_e32 v3, s7, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xor_i64_noret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB75_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB75_4 |
| ; GCN2-NEXT: .LBB75_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB75_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB75_2 |
| ; GCN2-NEXT: .LBB75_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s34 |
| ; GCN2-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_xor_b32_e32 v2, s6, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_xor_b32_e32 v3, s7, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xor_i64_noret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB75_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB75_4 |
| ; GCN3-NEXT: .LBB75_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB75_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB75_2 |
| ; GCN3-NEXT: .LBB75_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_xor_b32_e32 v1, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_xor_b32_e32 v2, s6, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_xor_i64_ret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB76_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB76_3 |
| ; GCN1-NEXT: s_branch .LBB76_4 |
| ; GCN1-NEXT: .LBB76_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB76_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_xor_b32_e32 v4, s6, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_xor_b32_e32 v5, s7, v1 |
| ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB76_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xor_i64_ret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB76_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB76_3 |
| ; GCN2-NEXT: s_branch .LBB76_4 |
| ; GCN2-NEXT: .LBB76_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB76_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_xor_b32_e32 v4, s6, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_xor_b32_e32 v5, s7, v1 |
| ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB76_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xor_i64_ret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB76_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB76_3 |
| ; GCN3-NEXT: s_branch .LBB76_4 |
| ; GCN3-NEXT: .LBB76_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB76_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_xor_b32_e32 v3, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_xor_b32_e32 v4, s6, v0 |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB76_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw xor ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_xor_i64_ret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB77_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB77_3 |
| ; GCN1-NEXT: s_branch .LBB77_4 |
| ; GCN1-NEXT: .LBB77_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB77_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_xor_b32_e32 v4, s6, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_xor_b32_e32 v5, s7, v1 |
| ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB77_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xor_i64_ret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB77_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB77_3 |
| ; GCN2-NEXT: s_branch .LBB77_4 |
| ; GCN2-NEXT: .LBB77_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB77_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_xor_b32_e32 v4, s6, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_xor_b32_e32 v5, s7, v1 |
| ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB77_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xor_i64_ret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB77_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB77_3 |
| ; GCN3-NEXT: s_branch .LBB77_4 |
| ; GCN3-NEXT: .LBB77_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB77_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_xor_b32_e32 v3, s7, v1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_xor_b32_e32 v4, s6, v0 |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB77_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw xor ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define void @flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB78_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB78_4 |
| ; GCN1-NEXT: .LBB78_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB78_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB78_2 |
| ; GCN1-NEXT: .LBB78_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_xor_b32_e32 v2, v4, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_xor_b32_e32 v3, v5, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB78_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB78_4 |
| ; GCN2-NEXT: .LBB78_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB78_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB78_2 |
| ; GCN2-NEXT: .LBB78_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_xor_b32_e32 v2, v4, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_xor_b32_e32 v3, v5, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB78_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB78_4 |
| ; GCN3-NEXT: .LBB78_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB78_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB78_2 |
| ; GCN3-NEXT: .LBB78_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_xor_b32_e32 v1, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_xor_b32_e32 v2, v4, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret void |
| } |
| |
| define i64 @flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB79_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB79_4 |
| ; GCN1-NEXT: .LBB79_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB79_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB79_2 |
| ; GCN1-NEXT: .LBB79_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_xor_b32_e32 v2, v0, v2 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_xor_b32_e32 v3, v1, v3 |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB79_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB79_4 |
| ; GCN2-NEXT: .LBB79_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB79_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB79_2 |
| ; GCN2-NEXT: .LBB79_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_xor_b32_e32 v2, v0, v2 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_xor_b32_e32 v3, v1, v3 |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB79_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB79_4 |
| ; GCN3-NEXT: .LBB79_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB79_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB79_2 |
| ; GCN3-NEXT: .LBB79_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_xor_b32_e32 v3, v1, v3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_xor_b32_e32 v2, v0, v2 |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw xor ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret i64 %result |
| } |
| |
| ; --------------------------------------------------------------------- |
| ; atomicrmw max |
| ; --------------------------------------------------------------------- |
| |
| define void @flat_atomic_max_i64_noret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_max_i64_noret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB80_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB80_6 |
| ; GCN1-NEXT: .LBB80_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB80_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB80_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB80_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB80_2 |
| ; GCN1-NEXT: .LBB80_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_max_i64_noret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB80_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB80_6 |
| ; GCN2-NEXT: .LBB80_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB80_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB80_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB80_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB80_2 |
| ; GCN2-NEXT: .LBB80_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_max_i64_noret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB80_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB80_6 |
| ; GCN3-NEXT: .LBB80_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB80_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB80_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB80_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB80_2 |
| ; GCN3-NEXT: .LBB80_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define void @flat_atomic_max_i64_noret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_max_i64_noret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB81_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB81_6 |
| ; GCN1-NEXT: .LBB81_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB81_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB81_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB81_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB81_2 |
| ; GCN1-NEXT: .LBB81_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_max_i64_noret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB81_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB81_6 |
| ; GCN2-NEXT: .LBB81_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB81_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB81_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB81_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB81_2 |
| ; GCN2-NEXT: .LBB81_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_max_i64_noret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB81_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB81_6 |
| ; GCN3-NEXT: .LBB81_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB81_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB81_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB81_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB81_2 |
| ; GCN3-NEXT: .LBB81_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define i64 @flat_atomic_max_i64_ret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_max_i64_ret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB82_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v5, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v4, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB82_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB82_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: .LBB82_4: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB82_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[4:5], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB82_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: v_mov_b32_e32 v0, v4 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, v5 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_max_i64_ret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB82_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v5, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v4, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB82_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB82_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: .LBB82_4: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB82_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[4:5], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB82_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: v_mov_b32_e32 v0, v4 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, v5 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_max_i64_ret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB82_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB82_6 |
| ; GCN3-NEXT: .LBB82_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB82_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB82_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB82_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB82_2 |
| ; GCN3-NEXT: .LBB82_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw max ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define i64 @flat_atomic_max_i64_ret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_max_i64_ret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB83_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB83_6 |
| ; GCN1-NEXT: .LBB83_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB83_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB83_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB83_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB83_2 |
| ; GCN1-NEXT: .LBB83_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_max_i64_ret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB83_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB83_6 |
| ; GCN2-NEXT: .LBB83_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB83_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB83_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB83_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB83_2 |
| ; GCN2-NEXT: .LBB83_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_max_i64_ret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB83_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB83_6 |
| ; GCN3-NEXT: .LBB83_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB83_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB83_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB83_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB83_2 |
| ; GCN3-NEXT: .LBB83_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw max ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_max_i64_noret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB84_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB84_6 |
| ; GCN1-NEXT: .LBB84_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB84_3: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN1-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN1-NEXT: .LBB84_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB84_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_branch .LBB84_2 |
| ; GCN1-NEXT: .LBB84_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_max_i64_noret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB84_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB84_6 |
| ; GCN2-NEXT: .LBB84_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB84_3: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN2-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN2-NEXT: .LBB84_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB84_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_branch .LBB84_2 |
| ; GCN2-NEXT: .LBB84_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_max_i64_noret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB84_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB84_6 |
| ; GCN3-NEXT: .LBB84_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB84_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN3-NEXT: .LBB84_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB84_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_branch .LBB84_2 |
| ; GCN3-NEXT: .LBB84_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_max_i64_noret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB85_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB85_6 |
| ; GCN1-NEXT: .LBB85_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB85_3: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN1-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN1-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN1-NEXT: .LBB85_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB85_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_branch .LBB85_2 |
| ; GCN1-NEXT: .LBB85_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_max_i64_noret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB85_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB85_6 |
| ; GCN2-NEXT: .LBB85_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB85_3: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN2-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN2-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN2-NEXT: .LBB85_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB85_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_branch .LBB85_2 |
| ; GCN2-NEXT: .LBB85_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_max_i64_noret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB85_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB85_6 |
| ; GCN3-NEXT: .LBB85_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB85_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN3-NEXT: .LBB85_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB85_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_branch .LBB85_2 |
| ; GCN3-NEXT: .LBB85_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_max_i64_ret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB86_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: .LBB86_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB86_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_branch .LBB86_6 |
| ; GCN1-NEXT: .LBB86_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB86_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB86_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_max_i64_ret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB86_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: .LBB86_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB86_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_branch .LBB86_6 |
| ; GCN2-NEXT: .LBB86_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB86_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB86_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_max_i64_ret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB86_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN3-NEXT: .LBB86_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB86_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_branch .LBB86_6 |
| ; GCN3-NEXT: .LBB86_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB86_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB86_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw max ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_max_i64_ret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB87_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN1-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: .LBB87_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB87_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_branch .LBB87_6 |
| ; GCN1-NEXT: .LBB87_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB87_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB87_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_max_i64_ret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB87_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN2-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: .LBB87_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB87_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_branch .LBB87_6 |
| ; GCN2-NEXT: .LBB87_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB87_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB87_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_max_i64_ret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB87_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN3-NEXT: .LBB87_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB87_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_branch .LBB87_6 |
| ; GCN3-NEXT: .LBB87_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB87_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB87_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw max ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %index) { |
| ; GCN1-LABEL: atomic_max_i64_addr64_offset: |
| ; GCN1: ; %bb.0: ; %entry |
| ; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 |
| ; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 |
| ; GCN1-NEXT: s_mov_b32 s14, -1 |
| ; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd |
| ; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 |
| ; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f |
| ; GCN1-NEXT: s_mov_b32 s15, 0xe8f000 |
| ; GCN1-NEXT: s_add_u32 s12, s12, s11 |
| ; GCN1-NEXT: s_addc_u32 s13, s13, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 |
| ; GCN1-NEXT: s_add_u32 s0, s0, s4 |
| ; GCN1-NEXT: s_addc_u32 s1, s1, s5 |
| ; GCN1-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN1-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN1-NEXT: s_cmp_eq_u32 s1, s8 |
| ; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB88_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow6 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB88_6 |
| ; GCN1-NEXT: .LBB88_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_endpgm |
| ; GCN1-NEXT: .LBB88_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN1-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN1-NEXT: .LBB88_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB88_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_branch .LBB88_2 |
| ; GCN1-NEXT: .LBB88_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s2 |
| ; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec |
| ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s3 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen |
| ; GCN1-NEXT: s_endpgm |
| ; |
| ; GCN2-LABEL: atomic_max_i64_addr64_offset: |
| ; GCN2: ; %bb.0: ; %entry |
| ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 |
| ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 |
| ; GCN2-NEXT: s_mov_b32 s90, -1 |
| ; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 |
| ; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 |
| ; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc |
| ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 |
| ; GCN2-NEXT: s_add_u32 s88, s88, s11 |
| ; GCN2-NEXT: s_addc_u32 s89, s89, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 |
| ; GCN2-NEXT: s_add_u32 s0, s0, s4 |
| ; GCN2-NEXT: s_addc_u32 s1, s1, s5 |
| ; GCN2-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN2-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN2-NEXT: s_cmp_eq_u32 s1, s8 |
| ; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB88_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow6 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB88_6 |
| ; GCN2-NEXT: .LBB88_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_endpgm |
| ; GCN2-NEXT: .LBB88_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN2-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN2-NEXT: .LBB88_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB88_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_branch .LBB88_2 |
| ; GCN2-NEXT: .LBB88_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s2 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s3 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: s_endpgm |
| ; |
| ; GCN3-LABEL: atomic_max_i64_addr64_offset: |
| ; GCN3: ; %bb.0: ; %entry |
| ; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 |
| ; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 |
| ; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 |
| ; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 |
| ; GCN3-NEXT: s_mov_b32 s14, -1 |
| ; GCN3-NEXT: s_mov_b32 s15, 0xe00000 |
| ; GCN3-NEXT: s_add_u32 s12, s12, s11 |
| ; GCN3-NEXT: s_addc_u32 s13, s13, 0 |
| ; GCN3-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN3-NEXT: s_lshl_b64 s[6:7], s[6:7], 3 |
| ; GCN3-NEXT: s_add_u32 s0, s0, s6 |
| ; GCN3-NEXT: s_addc_u32 s1, s1, s7 |
| ; GCN3-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN3-NEXT: s_cmp_eq_u32 s1, s5 |
| ; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB88_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow6 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB88_6 |
| ; GCN3-NEXT: .LBB88_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_endpgm |
| ; GCN3-NEXT: .LBB88_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN3-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN3-NEXT: .LBB88_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB88_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_branch .LBB88_2 |
| ; GCN3-NEXT: .LBB88_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s2 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4 |
| ; GCN3-NEXT: s_endpgm |
| entry: |
| %ptr = getelementptr i64, ptr %out, i64 %index |
| %gep = getelementptr i64, ptr %ptr, i64 4 |
| %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) { |
| ; GCN1-LABEL: atomic_max_i64_ret_addr64_offset: |
| ; GCN1: ; %bb.0: ; %entry |
| ; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 |
| ; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 |
| ; GCN1-NEXT: s_mov_b32 s18, -1 |
| ; GCN1-NEXT: s_mov_b32 s19, 0xe8f000 |
| ; GCN1-NEXT: s_add_u32 s16, s16, s11 |
| ; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9 |
| ; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41 |
| ; GCN1-NEXT: s_addc_u32 s17, s17, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN1-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN1-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN1-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN1-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN1-NEXT: s_cmp_eq_u32 s1, s2 |
| ; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB89_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN1-NEXT: .LBB89_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB89_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN1-NEXT: s_branch .LBB89_6 |
| ; GCN1-NEXT: .LBB89_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB89_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec |
| ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen |
| ; GCN1-NEXT: .LBB89_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN1-NEXT: s_endpgm |
| ; |
| ; GCN2-LABEL: atomic_max_i64_ret_addr64_offset: |
| ; GCN2: ; %bb.0: ; %entry |
| ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 |
| ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 |
| ; GCN2-NEXT: s_mov_b32 s90, -1 |
| ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 |
| ; GCN2-NEXT: s_add_u32 s88, s88, s11 |
| ; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 |
| ; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104 |
| ; GCN2-NEXT: s_addc_u32 s89, s89, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN2-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN2-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN2-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN2-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN2-NEXT: s_cmp_eq_u32 s1, s2 |
| ; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB89_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN2-NEXT: .LBB89_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB89_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN2-NEXT: s_branch .LBB89_6 |
| ; GCN2-NEXT: .LBB89_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB89_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: .LBB89_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN2-NEXT: s_endpgm |
| ; |
| ; GCN3-LABEL: atomic_max_i64_ret_addr64_offset: |
| ; GCN3: ; %bb.0: ; %entry |
| ; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 |
| ; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 |
| ; GCN3-NEXT: s_mov_b32 s18, -1 |
| ; GCN3-NEXT: s_mov_b32 s19, 0xe00000 |
| ; GCN3-NEXT: s_add_u32 s16, s16, s11 |
| ; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 |
| ; GCN3-NEXT: s_addc_u32 s17, s17, 0 |
| ; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base |
| ; GCN3-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN3-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN3-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN3-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN3-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN3-NEXT: s_cmp_eq_u32 s1, s3 |
| ; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB89_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN3-NEXT: .LBB89_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB89_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN3-NEXT: s_branch .LBB89_6 |
| ; GCN3-NEXT: .LBB89_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB89_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s12 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s13 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB89_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN3-NEXT: s_endpgm |
| entry: |
| %ptr = getelementptr i64, ptr %out, i64 %index |
| %gep = getelementptr i64, ptr %ptr, i64 4 |
| %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst |
| store i64 %tmp0, ptr %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index) { |
| ; GCN1-LABEL: atomic_max_i64_addr64: |
| ; GCN1: ; %bb.0: ; %entry |
| ; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 |
| ; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 |
| ; GCN1-NEXT: s_mov_b32 s14, -1 |
| ; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd |
| ; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 |
| ; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f |
| ; GCN1-NEXT: s_mov_b32 s15, 0xe8f000 |
| ; GCN1-NEXT: s_add_u32 s12, s12, s11 |
| ; GCN1-NEXT: s_addc_u32 s13, s13, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 |
| ; GCN1-NEXT: s_add_u32 s0, s0, s4 |
| ; GCN1-NEXT: s_addc_u32 s1, s1, s5 |
| ; GCN1-NEXT: s_cmp_eq_u32 s1, s8 |
| ; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB90_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow6 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB90_6 |
| ; GCN1-NEXT: .LBB90_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_endpgm |
| ; GCN1-NEXT: .LBB90_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN1-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN1-NEXT: .LBB90_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB90_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_branch .LBB90_2 |
| ; GCN1-NEXT: .LBB90_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s2 |
| ; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec |
| ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s3 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen |
| ; GCN1-NEXT: s_endpgm |
| ; |
| ; GCN2-LABEL: atomic_max_i64_addr64: |
| ; GCN2: ; %bb.0: ; %entry |
| ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 |
| ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 |
| ; GCN2-NEXT: s_mov_b32 s90, -1 |
| ; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 |
| ; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 |
| ; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc |
| ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 |
| ; GCN2-NEXT: s_add_u32 s88, s88, s11 |
| ; GCN2-NEXT: s_addc_u32 s89, s89, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 |
| ; GCN2-NEXT: s_add_u32 s0, s0, s4 |
| ; GCN2-NEXT: s_addc_u32 s1, s1, s5 |
| ; GCN2-NEXT: s_cmp_eq_u32 s1, s8 |
| ; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB90_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow6 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB90_6 |
| ; GCN2-NEXT: .LBB90_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_endpgm |
| ; GCN2-NEXT: .LBB90_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN2-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN2-NEXT: .LBB90_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB90_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_branch .LBB90_2 |
| ; GCN2-NEXT: .LBB90_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s2 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s3 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: s_endpgm |
| ; |
| ; GCN3-LABEL: atomic_max_i64_addr64: |
| ; GCN3: ; %bb.0: ; %entry |
| ; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 |
| ; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 |
| ; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 |
| ; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 |
| ; GCN3-NEXT: s_mov_b32 s14, -1 |
| ; GCN3-NEXT: s_mov_b32 s15, 0xe00000 |
| ; GCN3-NEXT: s_add_u32 s12, s12, s11 |
| ; GCN3-NEXT: s_addc_u32 s13, s13, 0 |
| ; GCN3-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN3-NEXT: s_lshl_b64 s[6:7], s[6:7], 3 |
| ; GCN3-NEXT: s_add_u32 s0, s0, s6 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: s_addc_u32 s1, s1, s7 |
| ; GCN3-NEXT: s_cmp_eq_u32 s1, s5 |
| ; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB90_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow6 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB90_6 |
| ; GCN3-NEXT: .LBB90_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_endpgm |
| ; GCN3-NEXT: .LBB90_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN3-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN3-NEXT: .LBB90_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB90_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_branch .LBB90_2 |
| ; GCN3-NEXT: .LBB90_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s2 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4 |
| ; GCN3-NEXT: s_endpgm |
| entry: |
| %ptr = getelementptr i64, ptr %out, i64 %index |
| %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) { |
| ; GCN1-LABEL: atomic_max_i64_ret_addr64: |
| ; GCN1: ; %bb.0: ; %entry |
| ; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 |
| ; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 |
| ; GCN1-NEXT: s_mov_b32 s18, -1 |
| ; GCN1-NEXT: s_mov_b32 s19, 0xe8f000 |
| ; GCN1-NEXT: s_add_u32 s16, s16, s11 |
| ; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9 |
| ; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41 |
| ; GCN1-NEXT: s_addc_u32 s17, s17, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN1-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN1-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN1-NEXT: s_cmp_eq_u32 s1, s2 |
| ; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB91_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN1-NEXT: .LBB91_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB91_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN1-NEXT: s_branch .LBB91_6 |
| ; GCN1-NEXT: .LBB91_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB91_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec |
| ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen |
| ; GCN1-NEXT: .LBB91_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN1-NEXT: s_endpgm |
| ; |
| ; GCN2-LABEL: atomic_max_i64_ret_addr64: |
| ; GCN2: ; %bb.0: ; %entry |
| ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 |
| ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 |
| ; GCN2-NEXT: s_mov_b32 s90, -1 |
| ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 |
| ; GCN2-NEXT: s_add_u32 s88, s88, s11 |
| ; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 |
| ; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104 |
| ; GCN2-NEXT: s_addc_u32 s89, s89, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN2-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN2-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN2-NEXT: s_cmp_eq_u32 s1, s2 |
| ; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB91_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN2-NEXT: .LBB91_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB91_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN2-NEXT: s_branch .LBB91_6 |
| ; GCN2-NEXT: .LBB91_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB91_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: .LBB91_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN2-NEXT: s_endpgm |
| ; |
| ; GCN3-LABEL: atomic_max_i64_ret_addr64: |
| ; GCN3: ; %bb.0: ; %entry |
| ; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 |
| ; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 |
| ; GCN3-NEXT: s_mov_b32 s18, -1 |
| ; GCN3-NEXT: s_mov_b32 s19, 0xe00000 |
| ; GCN3-NEXT: s_add_u32 s16, s16, s11 |
| ; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 |
| ; GCN3-NEXT: s_addc_u32 s17, s17, 0 |
| ; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base |
| ; GCN3-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN3-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN3-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN3-NEXT: s_cmp_eq_u32 s1, s3 |
| ; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB91_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN3-NEXT: .LBB91_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB91_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN3-NEXT: s_branch .LBB91_6 |
| ; GCN3-NEXT: .LBB91_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB91_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s12 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s13 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB91_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN3-NEXT: s_endpgm |
| entry: |
| %ptr = getelementptr i64, ptr %out, i64 %index |
| %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst |
| store i64 %tmp0, ptr %out2 |
| ret void |
| } |
| |
| define void @flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB92_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB92_6 |
| ; GCN1-NEXT: .LBB92_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB92_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB92_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB92_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB92_2 |
| ; GCN1-NEXT: .LBB92_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB92_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB92_6 |
| ; GCN2-NEXT: .LBB92_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB92_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB92_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB92_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB92_2 |
| ; GCN2-NEXT: .LBB92_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB92_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB92_6 |
| ; GCN3-NEXT: .LBB92_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB92_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB92_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB92_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB92_2 |
| ; GCN3-NEXT: .LBB92_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret void |
| } |
| |
| define i64 @flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB93_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB93_6 |
| ; GCN1-NEXT: .LBB93_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB93_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB93_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB93_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB93_2 |
| ; GCN1-NEXT: .LBB93_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB93_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB93_6 |
| ; GCN2-NEXT: .LBB93_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB93_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB93_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB93_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB93_2 |
| ; GCN2-NEXT: .LBB93_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB93_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB93_6 |
| ; GCN3-NEXT: .LBB93_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB93_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB93_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB93_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB93_2 |
| ; GCN3-NEXT: .LBB93_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw max ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret i64 %result |
| } |
| |
| ; --------------------------------------------------------------------- |
| ; atomicrmw umax |
| ; --------------------------------------------------------------------- |
| |
| define void @flat_atomic_umax_i64_noret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_umax_i64_noret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB94_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB94_6 |
| ; GCN1-NEXT: .LBB94_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB94_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB94_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB94_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB94_2 |
| ; GCN1-NEXT: .LBB94_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umax_i64_noret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB94_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB94_6 |
| ; GCN2-NEXT: .LBB94_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB94_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB94_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB94_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB94_2 |
| ; GCN2-NEXT: .LBB94_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umax_i64_noret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB94_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB94_6 |
| ; GCN3-NEXT: .LBB94_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB94_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB94_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB94_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB94_2 |
| ; GCN3-NEXT: .LBB94_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define void @flat_atomic_umax_i64_noret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_umax_i64_noret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB95_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB95_6 |
| ; GCN1-NEXT: .LBB95_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB95_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB95_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB95_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB95_2 |
| ; GCN1-NEXT: .LBB95_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umax_i64_noret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB95_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB95_6 |
| ; GCN2-NEXT: .LBB95_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB95_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB95_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB95_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB95_2 |
| ; GCN2-NEXT: .LBB95_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umax_i64_noret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB95_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB95_6 |
| ; GCN3-NEXT: .LBB95_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB95_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB95_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB95_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB95_2 |
| ; GCN3-NEXT: .LBB95_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define i64 @flat_atomic_umax_i64_ret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_umax_i64_ret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB96_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v5, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v4, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB96_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB96_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: .LBB96_4: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB96_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[4:5], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB96_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: v_mov_b32_e32 v0, v4 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, v5 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umax_i64_ret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB96_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v5, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v4, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB96_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB96_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: .LBB96_4: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB96_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[4:5], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB96_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: v_mov_b32_e32 v0, v4 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, v5 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umax_i64_ret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB96_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB96_6 |
| ; GCN3-NEXT: .LBB96_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB96_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB96_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB96_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB96_2 |
| ; GCN3-NEXT: .LBB96_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw umax ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define i64 @flat_atomic_umax_i64_ret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_umax_i64_ret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB97_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB97_6 |
| ; GCN1-NEXT: .LBB97_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB97_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB97_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB97_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB97_2 |
| ; GCN1-NEXT: .LBB97_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umax_i64_ret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB97_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB97_6 |
| ; GCN2-NEXT: .LBB97_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB97_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB97_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB97_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB97_2 |
| ; GCN2-NEXT: .LBB97_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umax_i64_ret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB97_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB97_6 |
| ; GCN3-NEXT: .LBB97_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB97_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB97_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB97_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB97_2 |
| ; GCN3-NEXT: .LBB97_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw umax ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_umax_i64_noret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB98_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB98_6 |
| ; GCN1-NEXT: .LBB98_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB98_3: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN1-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN1-NEXT: .LBB98_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB98_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_branch .LBB98_2 |
| ; GCN1-NEXT: .LBB98_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umax_i64_noret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB98_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB98_6 |
| ; GCN2-NEXT: .LBB98_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB98_3: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN2-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN2-NEXT: .LBB98_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB98_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_branch .LBB98_2 |
| ; GCN2-NEXT: .LBB98_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umax_i64_noret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB98_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB98_6 |
| ; GCN3-NEXT: .LBB98_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB98_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN3-NEXT: .LBB98_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB98_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_branch .LBB98_2 |
| ; GCN3-NEXT: .LBB98_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_umax_i64_noret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB99_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB99_6 |
| ; GCN1-NEXT: .LBB99_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB99_3: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN1-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN1-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN1-NEXT: .LBB99_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB99_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_branch .LBB99_2 |
| ; GCN1-NEXT: .LBB99_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umax_i64_noret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB99_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB99_6 |
| ; GCN2-NEXT: .LBB99_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB99_3: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN2-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN2-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN2-NEXT: .LBB99_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB99_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_branch .LBB99_2 |
| ; GCN2-NEXT: .LBB99_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umax_i64_noret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB99_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB99_6 |
| ; GCN3-NEXT: .LBB99_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB99_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN3-NEXT: .LBB99_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB99_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_branch .LBB99_2 |
| ; GCN3-NEXT: .LBB99_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_umax_i64_ret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB100_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: .LBB100_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB100_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_branch .LBB100_6 |
| ; GCN1-NEXT: .LBB100_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB100_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB100_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umax_i64_ret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB100_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: .LBB100_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB100_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_branch .LBB100_6 |
| ; GCN2-NEXT: .LBB100_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB100_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB100_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umax_i64_ret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB100_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN3-NEXT: .LBB100_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB100_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_branch .LBB100_6 |
| ; GCN3-NEXT: .LBB100_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB100_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB100_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw umax ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_umax_i64_ret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB101_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN1-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: .LBB101_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB101_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_branch .LBB101_6 |
| ; GCN1-NEXT: .LBB101_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB101_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB101_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umax_i64_ret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB101_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN2-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: .LBB101_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB101_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_branch .LBB101_6 |
| ; GCN2-NEXT: .LBB101_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB101_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB101_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umax_i64_ret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB101_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN3-NEXT: .LBB101_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB101_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_branch .LBB101_6 |
| ; GCN3-NEXT: .LBB101_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB101_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB101_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw umax ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64 %index) { |
| ; GCN1-LABEL: atomic_umax_i64_addr64_offset: |
| ; GCN1: ; %bb.0: ; %entry |
| ; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 |
| ; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 |
| ; GCN1-NEXT: s_mov_b32 s14, -1 |
| ; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd |
| ; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 |
| ; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f |
| ; GCN1-NEXT: s_mov_b32 s15, 0xe8f000 |
| ; GCN1-NEXT: s_add_u32 s12, s12, s11 |
| ; GCN1-NEXT: s_addc_u32 s13, s13, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 |
| ; GCN1-NEXT: s_add_u32 s0, s0, s4 |
| ; GCN1-NEXT: s_addc_u32 s1, s1, s5 |
| ; GCN1-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN1-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN1-NEXT: s_cmp_eq_u32 s1, s8 |
| ; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB102_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow6 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB102_6 |
| ; GCN1-NEXT: .LBB102_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_endpgm |
| ; GCN1-NEXT: .LBB102_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN1-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN1-NEXT: .LBB102_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB102_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_branch .LBB102_2 |
| ; GCN1-NEXT: .LBB102_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s2 |
| ; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec |
| ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s3 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen |
| ; GCN1-NEXT: s_endpgm |
| ; |
| ; GCN2-LABEL: atomic_umax_i64_addr64_offset: |
| ; GCN2: ; %bb.0: ; %entry |
| ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 |
| ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 |
| ; GCN2-NEXT: s_mov_b32 s90, -1 |
| ; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 |
| ; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 |
| ; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc |
| ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 |
| ; GCN2-NEXT: s_add_u32 s88, s88, s11 |
| ; GCN2-NEXT: s_addc_u32 s89, s89, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 |
| ; GCN2-NEXT: s_add_u32 s0, s0, s4 |
| ; GCN2-NEXT: s_addc_u32 s1, s1, s5 |
| ; GCN2-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN2-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN2-NEXT: s_cmp_eq_u32 s1, s8 |
| ; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB102_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow6 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB102_6 |
| ; GCN2-NEXT: .LBB102_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_endpgm |
| ; GCN2-NEXT: .LBB102_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN2-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN2-NEXT: .LBB102_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB102_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_branch .LBB102_2 |
| ; GCN2-NEXT: .LBB102_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s2 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s3 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: s_endpgm |
| ; |
| ; GCN3-LABEL: atomic_umax_i64_addr64_offset: |
| ; GCN3: ; %bb.0: ; %entry |
| ; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 |
| ; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 |
| ; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 |
| ; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 |
| ; GCN3-NEXT: s_mov_b32 s14, -1 |
| ; GCN3-NEXT: s_mov_b32 s15, 0xe00000 |
| ; GCN3-NEXT: s_add_u32 s12, s12, s11 |
| ; GCN3-NEXT: s_addc_u32 s13, s13, 0 |
| ; GCN3-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN3-NEXT: s_lshl_b64 s[6:7], s[6:7], 3 |
| ; GCN3-NEXT: s_add_u32 s0, s0, s6 |
| ; GCN3-NEXT: s_addc_u32 s1, s1, s7 |
| ; GCN3-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN3-NEXT: s_cmp_eq_u32 s1, s5 |
| ; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB102_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow6 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB102_6 |
| ; GCN3-NEXT: .LBB102_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_endpgm |
| ; GCN3-NEXT: .LBB102_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN3-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN3-NEXT: .LBB102_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB102_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_branch .LBB102_2 |
| ; GCN3-NEXT: .LBB102_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s2 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4 |
| ; GCN3-NEXT: s_endpgm |
| entry: |
| %ptr = getelementptr i64, ptr %out, i64 %index |
| %gep = getelementptr i64, ptr %ptr, i64 4 |
| %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) { |
| ; GCN1-LABEL: atomic_umax_i64_ret_addr64_offset: |
| ; GCN1: ; %bb.0: ; %entry |
| ; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 |
| ; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 |
| ; GCN1-NEXT: s_mov_b32 s18, -1 |
| ; GCN1-NEXT: s_mov_b32 s19, 0xe8f000 |
| ; GCN1-NEXT: s_add_u32 s16, s16, s11 |
| ; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9 |
| ; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41 |
| ; GCN1-NEXT: s_addc_u32 s17, s17, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN1-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN1-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN1-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN1-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN1-NEXT: s_cmp_eq_u32 s1, s2 |
| ; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB103_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN1-NEXT: .LBB103_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB103_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN1-NEXT: s_branch .LBB103_6 |
| ; GCN1-NEXT: .LBB103_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB103_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec |
| ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen |
| ; GCN1-NEXT: .LBB103_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN1-NEXT: s_endpgm |
| ; |
| ; GCN2-LABEL: atomic_umax_i64_ret_addr64_offset: |
| ; GCN2: ; %bb.0: ; %entry |
| ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 |
| ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 |
| ; GCN2-NEXT: s_mov_b32 s90, -1 |
| ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 |
| ; GCN2-NEXT: s_add_u32 s88, s88, s11 |
| ; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 |
| ; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104 |
| ; GCN2-NEXT: s_addc_u32 s89, s89, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN2-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN2-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN2-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN2-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN2-NEXT: s_cmp_eq_u32 s1, s2 |
| ; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB103_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN2-NEXT: .LBB103_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB103_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN2-NEXT: s_branch .LBB103_6 |
| ; GCN2-NEXT: .LBB103_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB103_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: .LBB103_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN2-NEXT: s_endpgm |
| ; |
| ; GCN3-LABEL: atomic_umax_i64_ret_addr64_offset: |
| ; GCN3: ; %bb.0: ; %entry |
| ; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 |
| ; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 |
| ; GCN3-NEXT: s_mov_b32 s18, -1 |
| ; GCN3-NEXT: s_mov_b32 s19, 0xe00000 |
| ; GCN3-NEXT: s_add_u32 s16, s16, s11 |
| ; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 |
| ; GCN3-NEXT: s_addc_u32 s17, s17, 0 |
| ; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base |
| ; GCN3-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN3-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN3-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN3-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN3-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN3-NEXT: s_cmp_eq_u32 s1, s3 |
| ; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB103_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN3-NEXT: .LBB103_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB103_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN3-NEXT: s_branch .LBB103_6 |
| ; GCN3-NEXT: .LBB103_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB103_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s12 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s13 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB103_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN3-NEXT: s_endpgm |
| entry: |
| %ptr = getelementptr i64, ptr %out, i64 %index |
| %gep = getelementptr i64, ptr %ptr, i64 4 |
| %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst |
| store i64 %tmp0, ptr %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) { |
| ; GCN1-LABEL: atomic_umax_i64_ret_addr64: |
| ; GCN1: ; %bb.0: ; %entry |
| ; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 |
| ; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 |
| ; GCN1-NEXT: s_mov_b32 s18, -1 |
| ; GCN1-NEXT: s_mov_b32 s19, 0xe8f000 |
| ; GCN1-NEXT: s_add_u32 s16, s16, s11 |
| ; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9 |
| ; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41 |
| ; GCN1-NEXT: s_addc_u32 s17, s17, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN1-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN1-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN1-NEXT: s_cmp_eq_u32 s1, s2 |
| ; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB104_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN1-NEXT: .LBB104_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB104_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN1-NEXT: s_branch .LBB104_6 |
| ; GCN1-NEXT: .LBB104_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB104_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec |
| ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen |
| ; GCN1-NEXT: .LBB104_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN1-NEXT: s_endpgm |
| ; |
| ; GCN2-LABEL: atomic_umax_i64_ret_addr64: |
| ; GCN2: ; %bb.0: ; %entry |
| ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 |
| ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 |
| ; GCN2-NEXT: s_mov_b32 s90, -1 |
| ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 |
| ; GCN2-NEXT: s_add_u32 s88, s88, s11 |
| ; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 |
| ; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104 |
| ; GCN2-NEXT: s_addc_u32 s89, s89, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN2-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN2-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN2-NEXT: s_cmp_eq_u32 s1, s2 |
| ; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB104_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN2-NEXT: .LBB104_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB104_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN2-NEXT: s_branch .LBB104_6 |
| ; GCN2-NEXT: .LBB104_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB104_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: .LBB104_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN2-NEXT: s_endpgm |
| ; |
| ; GCN3-LABEL: atomic_umax_i64_ret_addr64: |
| ; GCN3: ; %bb.0: ; %entry |
| ; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 |
| ; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 |
| ; GCN3-NEXT: s_mov_b32 s18, -1 |
| ; GCN3-NEXT: s_mov_b32 s19, 0xe00000 |
| ; GCN3-NEXT: s_add_u32 s16, s16, s11 |
| ; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 |
| ; GCN3-NEXT: s_addc_u32 s17, s17, 0 |
| ; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base |
| ; GCN3-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN3-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN3-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN3-NEXT: s_cmp_eq_u32 s1, s3 |
| ; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB104_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN3-NEXT: .LBB104_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB104_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN3-NEXT: s_branch .LBB104_6 |
| ; GCN3-NEXT: .LBB104_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB104_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s12 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s13 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB104_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN3-NEXT: s_endpgm |
| entry: |
| %ptr = getelementptr i64, ptr %out, i64 %index |
| %tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst |
| store i64 %tmp0, ptr %out2 |
| ret void |
| } |
| |
| define void @flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB105_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB105_6 |
| ; GCN1-NEXT: .LBB105_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB105_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB105_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB105_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB105_2 |
| ; GCN1-NEXT: .LBB105_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB105_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB105_6 |
| ; GCN2-NEXT: .LBB105_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB105_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB105_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB105_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB105_2 |
| ; GCN2-NEXT: .LBB105_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB105_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB105_6 |
| ; GCN3-NEXT: .LBB105_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB105_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB105_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB105_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB105_2 |
| ; GCN3-NEXT: .LBB105_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret void |
| } |
| |
| define i64 @flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB106_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB106_6 |
| ; GCN1-NEXT: .LBB106_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB106_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB106_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB106_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB106_2 |
| ; GCN1-NEXT: .LBB106_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB106_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB106_6 |
| ; GCN2-NEXT: .LBB106_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB106_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB106_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB106_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB106_2 |
| ; GCN2-NEXT: .LBB106_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB106_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB106_6 |
| ; GCN3-NEXT: .LBB106_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB106_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB106_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB106_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB106_2 |
| ; GCN3-NEXT: .LBB106_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw umax ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret i64 %result |
| } |
| |
| ; --------------------------------------------------------------------- |
| ; atomicrmw umin |
| ; --------------------------------------------------------------------- |
| |
| define void @flat_atomic_umin_i64_noret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_umin_i64_noret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB107_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB107_6 |
| ; GCN1-NEXT: .LBB107_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB107_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB107_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB107_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB107_2 |
| ; GCN1-NEXT: .LBB107_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umin_i64_noret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB107_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB107_6 |
| ; GCN2-NEXT: .LBB107_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB107_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB107_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB107_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB107_2 |
| ; GCN2-NEXT: .LBB107_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umin_i64_noret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB107_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB107_6 |
| ; GCN3-NEXT: .LBB107_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB107_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB107_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB107_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB107_2 |
| ; GCN3-NEXT: .LBB107_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw umin ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define void @flat_atomic_umin_i64_noret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_umin_i64_noret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB108_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB108_6 |
| ; GCN1-NEXT: .LBB108_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB108_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB108_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB108_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB108_2 |
| ; GCN1-NEXT: .LBB108_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umin_i64_noret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB108_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB108_6 |
| ; GCN2-NEXT: .LBB108_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB108_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB108_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB108_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB108_2 |
| ; GCN2-NEXT: .LBB108_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umin_i64_noret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB108_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB108_6 |
| ; GCN3-NEXT: .LBB108_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB108_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB108_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB108_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB108_2 |
| ; GCN3-NEXT: .LBB108_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define i64 @flat_atomic_umin_i64_ret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_umin_i64_ret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB109_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v5, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v4, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB109_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB109_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: .LBB109_4: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB109_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[4:5], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB109_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: v_mov_b32_e32 v0, v4 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, v5 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umin_i64_ret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB109_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v5, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v4, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB109_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB109_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: .LBB109_4: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB109_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[4:5], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB109_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: v_mov_b32_e32 v0, v4 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, v5 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umin_i64_ret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB109_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB109_6 |
| ; GCN3-NEXT: .LBB109_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB109_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB109_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB109_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB109_2 |
| ; GCN3-NEXT: .LBB109_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw umin ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define i64 @flat_atomic_umin_i64_ret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_umin_i64_ret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB110_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB110_6 |
| ; GCN1-NEXT: .LBB110_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB110_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB110_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB110_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB110_2 |
| ; GCN1-NEXT: .LBB110_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umin_i64_ret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB110_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB110_6 |
| ; GCN2-NEXT: .LBB110_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB110_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB110_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB110_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB110_2 |
| ; GCN2-NEXT: .LBB110_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umin_i64_ret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB110_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB110_6 |
| ; GCN3-NEXT: .LBB110_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB110_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB110_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB110_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB110_2 |
| ; GCN3-NEXT: .LBB110_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw umin ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_umin_i64_noret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB111_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB111_6 |
| ; GCN1-NEXT: .LBB111_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB111_3: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN1-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN1-NEXT: .LBB111_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB111_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_branch .LBB111_2 |
| ; GCN1-NEXT: .LBB111_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umin_i64_noret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB111_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB111_6 |
| ; GCN2-NEXT: .LBB111_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB111_3: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN2-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN2-NEXT: .LBB111_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB111_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_branch .LBB111_2 |
| ; GCN2-NEXT: .LBB111_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umin_i64_noret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB111_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB111_6 |
| ; GCN3-NEXT: .LBB111_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB111_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN3-NEXT: .LBB111_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB111_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_branch .LBB111_2 |
| ; GCN3-NEXT: .LBB111_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw umin ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_umin_i64_noret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB112_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB112_6 |
| ; GCN1-NEXT: .LBB112_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB112_3: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN1-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN1-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN1-NEXT: .LBB112_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB112_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_branch .LBB112_2 |
| ; GCN1-NEXT: .LBB112_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umin_i64_noret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB112_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB112_6 |
| ; GCN2-NEXT: .LBB112_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB112_3: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN2-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN2-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN2-NEXT: .LBB112_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB112_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_branch .LBB112_2 |
| ; GCN2-NEXT: .LBB112_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umin_i64_noret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB112_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB112_6 |
| ; GCN3-NEXT: .LBB112_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB112_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN3-NEXT: .LBB112_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB112_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_branch .LBB112_2 |
| ; GCN3-NEXT: .LBB112_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_umin_i64_ret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB113_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: .LBB113_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB113_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_branch .LBB113_6 |
| ; GCN1-NEXT: .LBB113_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB113_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB113_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umin_i64_ret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB113_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: .LBB113_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB113_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_branch .LBB113_6 |
| ; GCN2-NEXT: .LBB113_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB113_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB113_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umin_i64_ret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB113_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN3-NEXT: .LBB113_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB113_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_branch .LBB113_6 |
| ; GCN3-NEXT: .LBB113_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB113_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB113_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw umin ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_umin_i64_ret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB114_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN1-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: .LBB114_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB114_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_branch .LBB114_6 |
| ; GCN1-NEXT: .LBB114_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB114_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB114_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umin_i64_ret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB114_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN2-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: .LBB114_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB114_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_branch .LBB114_6 |
| ; GCN2-NEXT: .LBB114_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB114_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB114_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umin_i64_ret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB114_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN3-NEXT: .LBB114_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB114_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_branch .LBB114_6 |
| ; GCN3-NEXT: .LBB114_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB114_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB114_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw umin ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define void @flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB115_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB115_6 |
| ; GCN1-NEXT: .LBB115_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB115_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB115_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB115_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB115_2 |
| ; GCN1-NEXT: .LBB115_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB115_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB115_6 |
| ; GCN2-NEXT: .LBB115_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB115_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB115_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB115_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB115_2 |
| ; GCN2-NEXT: .LBB115_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB115_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB115_6 |
| ; GCN3-NEXT: .LBB115_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB115_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB115_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB115_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB115_2 |
| ; GCN3-NEXT: .LBB115_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret void |
| } |
| |
| define i64 @flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB116_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB116_6 |
| ; GCN1-NEXT: .LBB116_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB116_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB116_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB116_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB116_2 |
| ; GCN1-NEXT: .LBB116_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB116_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB116_6 |
| ; GCN2-NEXT: .LBB116_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB116_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB116_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB116_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB116_2 |
| ; GCN2-NEXT: .LBB116_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB116_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB116_6 |
| ; GCN3-NEXT: .LBB116_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB116_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB116_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB116_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB116_2 |
| ; GCN3-NEXT: .LBB116_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw umin ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret i64 %result |
| } |
| |
| ; --------------------------------------------------------------------- |
| ; atomicrmw min |
| ; --------------------------------------------------------------------- |
| |
| define void @flat_atomic_min_i64_noret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_min_i64_noret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB117_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB117_6 |
| ; GCN1-NEXT: .LBB117_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB117_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB117_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB117_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB117_2 |
| ; GCN1-NEXT: .LBB117_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_min_i64_noret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB117_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB117_6 |
| ; GCN2-NEXT: .LBB117_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB117_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB117_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB117_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB117_2 |
| ; GCN2-NEXT: .LBB117_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_min_i64_noret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB117_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB117_6 |
| ; GCN3-NEXT: .LBB117_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB117_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB117_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB117_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB117_2 |
| ; GCN3-NEXT: .LBB117_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define void @flat_atomic_min_i64_noret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_min_i64_noret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB118_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB118_6 |
| ; GCN1-NEXT: .LBB118_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB118_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB118_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB118_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB118_2 |
| ; GCN1-NEXT: .LBB118_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_min_i64_noret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB118_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB118_6 |
| ; GCN2-NEXT: .LBB118_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB118_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB118_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB118_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB118_2 |
| ; GCN2-NEXT: .LBB118_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_min_i64_noret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB118_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB118_6 |
| ; GCN3-NEXT: .LBB118_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB118_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB118_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB118_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB118_2 |
| ; GCN3-NEXT: .LBB118_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define i64 @flat_atomic_min_i64_ret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_min_i64_ret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB119_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v5, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v4, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB119_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB119_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: .LBB119_4: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB119_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v1, vcc, 4, v0 |
| ; GCN1-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[4:5], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB119_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: v_mov_b32_e32 v0, v4 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, v5 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_min_i64_ret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB119_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v5, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v4, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB119_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB119_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: .LBB119_4: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB119_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v1, vcc, 4, v0 |
| ; GCN2-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[4:5], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB119_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: v_mov_b32_e32 v0, v4 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, v5 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_min_i64_ret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB119_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB119_6 |
| ; GCN3-NEXT: .LBB119_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB119_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB119_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB119_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB119_2 |
| ; GCN3-NEXT: .LBB119_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw min ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define i64 @flat_atomic_min_i64_ret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_min_i64_ret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB120_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB120_6 |
| ; GCN1-NEXT: .LBB120_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB120_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB120_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB120_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB120_2 |
| ; GCN1-NEXT: .LBB120_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_min_i64_ret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB120_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB120_6 |
| ; GCN2-NEXT: .LBB120_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB120_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB120_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB120_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB120_2 |
| ; GCN2-NEXT: .LBB120_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_min_i64_ret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB120_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB120_6 |
| ; GCN3-NEXT: .LBB120_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB120_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB120_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB120_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB120_2 |
| ; GCN3-NEXT: .LBB120_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw min ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_min_i64_noret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB121_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB121_6 |
| ; GCN1-NEXT: .LBB121_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB121_3: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN1-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN1-NEXT: .LBB121_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB121_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_branch .LBB121_2 |
| ; GCN1-NEXT: .LBB121_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_min_i64_noret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB121_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB121_6 |
| ; GCN2-NEXT: .LBB121_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB121_3: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN2-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN2-NEXT: .LBB121_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB121_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_branch .LBB121_2 |
| ; GCN2-NEXT: .LBB121_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_min_i64_noret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB121_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB121_6 |
| ; GCN3-NEXT: .LBB121_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB121_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s5 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN3-NEXT: .LBB121_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB121_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_branch .LBB121_2 |
| ; GCN3-NEXT: .LBB121_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_min_i64_noret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB122_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB122_6 |
| ; GCN1-NEXT: .LBB122_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB122_3: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN1-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN1-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN1-NEXT: .LBB122_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB122_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_branch .LBB122_2 |
| ; GCN1-NEXT: .LBB122_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_min_i64_noret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB122_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB122_6 |
| ; GCN2-NEXT: .LBB122_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB122_3: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN2-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN2-NEXT: flat_load_dword v3, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v2, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN2-NEXT: .LBB122_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB122_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_branch .LBB122_2 |
| ; GCN2-NEXT: .LBB122_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_min_i64_noret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB122_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB122_6 |
| ; GCN3-NEXT: .LBB122_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB122_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s35 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v7, s6 |
| ; GCN3-NEXT: .LBB122_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB122_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_branch .LBB122_2 |
| ; GCN3-NEXT: .LBB122_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_min_i64_ret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB123_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: .LBB123_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB123_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN1-NEXT: s_branch .LBB123_6 |
| ; GCN1-NEXT: .LBB123_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB123_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB123_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_min_i64_ret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB123_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s34, s4, 4 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: .LBB123_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB123_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN2-NEXT: s_branch .LBB123_6 |
| ; GCN2-NEXT: .LBB123_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB123_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB123_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_min_i64_ret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB123_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s5 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN3-NEXT: .LBB123_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB123_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[34:35] |
| ; GCN3-NEXT: s_branch .LBB123_6 |
| ; GCN3-NEXT: .LBB123_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB123_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB123_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw min ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_min_i64_ret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB124_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN1-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: .LBB124_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB124_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN1-NEXT: s_branch .LBB124_6 |
| ; GCN1-NEXT: .LBB124_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB124_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB124_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_min_i64_ret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB124_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: s_add_u32 s36, s34, 4 |
| ; GCN2-NEXT: s_addc_u32 s37, s35, 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s36 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s37 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: .LBB124_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB124_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN2-NEXT: s_branch .LBB124_6 |
| ; GCN2-NEXT: .LBB124_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB124_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB124_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_min_i64_ret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB124_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s35 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN3-NEXT: .LBB124_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB124_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[36:37] |
| ; GCN3-NEXT: s_branch .LBB124_6 |
| ; GCN3-NEXT: .LBB124_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB124_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB124_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw min ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %index) { |
| ; GCN1-LABEL: atomic_min_i64_addr64_offset: |
| ; GCN1: ; %bb.0: ; %entry |
| ; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 |
| ; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 |
| ; GCN1-NEXT: s_mov_b32 s14, -1 |
| ; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd |
| ; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 |
| ; GCN1-NEXT: s_load_dword s8, s[4:5], 0x3f |
| ; GCN1-NEXT: s_mov_b32 s15, 0xe8f000 |
| ; GCN1-NEXT: s_add_u32 s12, s12, s11 |
| ; GCN1-NEXT: s_addc_u32 s13, s13, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 |
| ; GCN1-NEXT: s_add_u32 s0, s0, s4 |
| ; GCN1-NEXT: s_addc_u32 s1, s1, s5 |
| ; GCN1-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN1-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN1-NEXT: s_cmp_eq_u32 s1, s8 |
| ; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB125_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow6 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB125_6 |
| ; GCN1-NEXT: .LBB125_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_endpgm |
| ; GCN1-NEXT: .LBB125_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN1-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN1-NEXT: .LBB125_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB125_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_branch .LBB125_2 |
| ; GCN1-NEXT: .LBB125_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s2 |
| ; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec |
| ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s3 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen |
| ; GCN1-NEXT: s_endpgm |
| ; |
| ; GCN2-LABEL: atomic_min_i64_addr64_offset: |
| ; GCN2: ; %bb.0: ; %entry |
| ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 |
| ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 |
| ; GCN2-NEXT: s_mov_b32 s90, -1 |
| ; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 |
| ; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 |
| ; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc |
| ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 |
| ; GCN2-NEXT: s_add_u32 s88, s88, s11 |
| ; GCN2-NEXT: s_addc_u32 s89, s89, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3 |
| ; GCN2-NEXT: s_add_u32 s0, s0, s4 |
| ; GCN2-NEXT: s_addc_u32 s1, s1, s5 |
| ; GCN2-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN2-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN2-NEXT: s_cmp_eq_u32 s1, s8 |
| ; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB125_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow6 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB125_6 |
| ; GCN2-NEXT: .LBB125_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_endpgm |
| ; GCN2-NEXT: .LBB125_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN2-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN2-NEXT: .LBB125_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB125_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_branch .LBB125_2 |
| ; GCN2-NEXT: .LBB125_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s2 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s3 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: s_endpgm |
| ; |
| ; GCN3-LABEL: atomic_min_i64_addr64_offset: |
| ; GCN3: ; %bb.0: ; %entry |
| ; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 |
| ; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 |
| ; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34 |
| ; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 |
| ; GCN3-NEXT: s_mov_b32 s14, -1 |
| ; GCN3-NEXT: s_mov_b32 s15, 0xe00000 |
| ; GCN3-NEXT: s_add_u32 s12, s12, s11 |
| ; GCN3-NEXT: s_addc_u32 s13, s13, 0 |
| ; GCN3-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN3-NEXT: s_lshl_b64 s[6:7], s[6:7], 3 |
| ; GCN3-NEXT: s_add_u32 s0, s0, s6 |
| ; GCN3-NEXT: s_addc_u32 s1, s1, s7 |
| ; GCN3-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN3-NEXT: s_cmp_eq_u32 s1, s5 |
| ; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB125_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow6 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB125_6 |
| ; GCN3-NEXT: .LBB125_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_endpgm |
| ; GCN3-NEXT: .LBB125_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN3-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN3-NEXT: .LBB125_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB125_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_branch .LBB125_2 |
| ; GCN3-NEXT: .LBB125_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s2 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4 |
| ; GCN3-NEXT: s_endpgm |
| entry: |
| %ptr = getelementptr i64, ptr %out, i64 %index |
| %gep = getelementptr i64, ptr %ptr, i64 4 |
| %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) { |
| ; GCN1-LABEL: atomic_min_i64_ret_addr64_offset: |
| ; GCN1: ; %bb.0: ; %entry |
| ; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 |
| ; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 |
| ; GCN1-NEXT: s_mov_b32 s18, -1 |
| ; GCN1-NEXT: s_mov_b32 s19, 0xe8f000 |
| ; GCN1-NEXT: s_add_u32 s16, s16, s11 |
| ; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9 |
| ; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41 |
| ; GCN1-NEXT: s_addc_u32 s17, s17, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN1-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN1-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN1-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN1-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN1-NEXT: s_cmp_eq_u32 s1, s2 |
| ; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB126_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN1-NEXT: .LBB126_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB126_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN1-NEXT: s_branch .LBB126_6 |
| ; GCN1-NEXT: .LBB126_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB126_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec |
| ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen |
| ; GCN1-NEXT: .LBB126_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN1-NEXT: s_endpgm |
| ; |
| ; GCN2-LABEL: atomic_min_i64_ret_addr64_offset: |
| ; GCN2: ; %bb.0: ; %entry |
| ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 |
| ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 |
| ; GCN2-NEXT: s_mov_b32 s90, -1 |
| ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 |
| ; GCN2-NEXT: s_add_u32 s88, s88, s11 |
| ; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 |
| ; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104 |
| ; GCN2-NEXT: s_addc_u32 s89, s89, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN2-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN2-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN2-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN2-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN2-NEXT: s_cmp_eq_u32 s1, s2 |
| ; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB126_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN2-NEXT: .LBB126_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB126_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN2-NEXT: s_branch .LBB126_6 |
| ; GCN2-NEXT: .LBB126_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB126_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: .LBB126_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN2-NEXT: s_endpgm |
| ; |
| ; GCN3-LABEL: atomic_min_i64_ret_addr64_offset: |
| ; GCN3: ; %bb.0: ; %entry |
| ; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 |
| ; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 |
| ; GCN3-NEXT: s_mov_b32 s18, -1 |
| ; GCN3-NEXT: s_mov_b32 s19, 0xe00000 |
| ; GCN3-NEXT: s_add_u32 s16, s16, s11 |
| ; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 |
| ; GCN3-NEXT: s_addc_u32 s17, s17, 0 |
| ; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base |
| ; GCN3-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN3-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN3-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN3-NEXT: s_add_u32 s0, s0, 32 |
| ; GCN3-NEXT: s_addc_u32 s1, s1, 0 |
| ; GCN3-NEXT: s_cmp_eq_u32 s1, s3 |
| ; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB126_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN3-NEXT: .LBB126_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB126_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN3-NEXT: s_branch .LBB126_6 |
| ; GCN3-NEXT: .LBB126_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB126_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s12 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s13 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB126_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN3-NEXT: s_endpgm |
| entry: |
| %ptr = getelementptr i64, ptr %out, i64 %index |
| %gep = getelementptr i64, ptr %ptr, i64 4 |
| %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst |
| store i64 %tmp0, ptr %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) { |
| ; GCN1-LABEL: atomic_min_i64: |
| ; GCN1: ; %bb.0: ; %entry |
| ; GCN1-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 |
| ; GCN1-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 |
| ; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x3d |
| ; GCN1-NEXT: s_mov_b32 s14, -1 |
| ; GCN1-NEXT: s_mov_b32 s15, 0xe8f000 |
| ; GCN1-NEXT: s_add_u32 s12, s12, s11 |
| ; GCN1-NEXT: s_addc_u32 s13, s13, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s1, s4 |
| ; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB127_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow5 |
| ; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB127_6 |
| ; GCN1-NEXT: .LBB127_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_endpgm |
| ; GCN1-NEXT: .LBB127_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN1-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN1-NEXT: .LBB127_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN1-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB127_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_branch .LBB127_2 |
| ; GCN1-NEXT: .LBB127_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s2 |
| ; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec |
| ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[12:15], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s3 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[12:15], 0 offen |
| ; GCN1-NEXT: s_endpgm |
| ; |
| ; GCN2-LABEL: atomic_min_i64: |
| ; GCN2: ; %bb.0: ; %entry |
| ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 |
| ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 |
| ; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0xf4 |
| ; GCN2-NEXT: s_mov_b32 s90, -1 |
| ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 |
| ; GCN2-NEXT: s_add_u32 s88, s88, s11 |
| ; GCN2-NEXT: s_addc_u32 s89, s89, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s1, s4 |
| ; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB127_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow5 |
| ; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB127_6 |
| ; GCN2-NEXT: .LBB127_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_endpgm |
| ; GCN2-NEXT: .LBB127_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN2-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN2-NEXT: .LBB127_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN2-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB127_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_branch .LBB127_2 |
| ; GCN2-NEXT: .LBB127_6: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s2 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s3 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: s_endpgm |
| ; |
| ; GCN3-LABEL: atomic_min_i64: |
| ; GCN3: ; %bb.0: ; %entry |
| ; GCN3-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 |
| ; GCN3-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 |
| ; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 |
| ; GCN3-NEXT: s_mov_b32 s14, -1 |
| ; GCN3-NEXT: s_mov_b32 s15, 0xe00000 |
| ; GCN3-NEXT: s_add_u32 s12, s12, s11 |
| ; GCN3-NEXT: s_addc_u32 s13, s13, 0 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN3-NEXT: s_cmp_eq_u32 s1, s5 |
| ; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[4:5], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB127_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow5 |
| ; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB127_6 |
| ; GCN3-NEXT: .LBB127_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_endpgm |
| ; GCN3-NEXT: .LBB127_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s1 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s0 |
| ; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[4:5], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v6, s3 |
| ; GCN3-NEXT: v_mov_b32_e32 v7, s2 |
| ; GCN3-NEXT: .LBB127_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_mov_b32_e32 v3, v1 |
| ; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GCN3-NEXT: v_mov_b32_e32 v2, v0 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB127_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_branch .LBB127_2 |
| ; GCN3-NEXT: .LBB127_6: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[12:15], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[12:15], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s2 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s3 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[12:15], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[12:15], 0 offen offset:4 |
| ; GCN3-NEXT: s_endpgm |
| entry: |
| %tmp0 = atomicrmw min ptr %out, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) { |
| ; GCN1-LABEL: atomic_min_i64_ret_addr64: |
| ; GCN1: ; %bb.0: ; %entry |
| ; GCN1-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 |
| ; GCN1-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 |
| ; GCN1-NEXT: s_mov_b32 s18, -1 |
| ; GCN1-NEXT: s_mov_b32 s19, 0xe8f000 |
| ; GCN1-NEXT: s_add_u32 s16, s16, s11 |
| ; GCN1-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9 |
| ; GCN1-NEXT: s_load_dword s2, s[4:5], 0x41 |
| ; GCN1-NEXT: s_addc_u32 s17, s17, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN1-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN1-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN1-NEXT: s_cmp_eq_u32 s1, s2 |
| ; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB128_4 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN1-NEXT: .LBB128_2: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB128_2 |
| ; GCN1-NEXT: ; %bb.3: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN1-NEXT: s_branch .LBB128_6 |
| ; GCN1-NEXT: .LBB128_4: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_cbranch_execz .LBB128_6 |
| ; GCN1-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec |
| ; GCN1-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN1-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[16:19], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[16:19], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[16:19], 0 offen |
| ; GCN1-NEXT: .LBB128_6: ; %atomicrmw.phi |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN1-NEXT: s_endpgm |
| ; |
| ; GCN2-LABEL: atomic_min_i64_ret_addr64: |
| ; GCN2: ; %bb.0: ; %entry |
| ; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0 |
| ; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1 |
| ; GCN2-NEXT: s_mov_b32 s90, -1 |
| ; GCN2-NEXT: s_mov_b32 s91, 0xe80000 |
| ; GCN2-NEXT: s_add_u32 s88, s88, s11 |
| ; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 |
| ; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104 |
| ; GCN2-NEXT: s_addc_u32 s89, s89, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN2-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN2-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN2-NEXT: s_cmp_eq_u32 s1, s2 |
| ; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB128_4 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN2-NEXT: .LBB128_2: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB128_2 |
| ; GCN2-NEXT: ; %bb.3: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN2-NEXT: s_branch .LBB128_6 |
| ; GCN2-NEXT: .LBB128_4: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_cbranch_execz .LBB128_6 |
| ; GCN2-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN2-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN2-NEXT: s_add_i32 s0, s0, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s0 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v4, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[88:91], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[88:91], 0 offen |
| ; GCN2-NEXT: .LBB128_6: ; %atomicrmw.phi |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN2-NEXT: s_endpgm |
| ; |
| ; GCN3-LABEL: atomic_min_i64_ret_addr64: |
| ; GCN3: ; %bb.0: ; %entry |
| ; GCN3-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 |
| ; GCN3-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 |
| ; GCN3-NEXT: s_mov_b32 s18, -1 |
| ; GCN3-NEXT: s_mov_b32 s19, 0xe00000 |
| ; GCN3-NEXT: s_add_u32 s16, s16, s11 |
| ; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24 |
| ; GCN3-NEXT: s_addc_u32 s17, s17, 0 |
| ; GCN3-NEXT: s_mov_b64 s[2:3], src_private_base |
| ; GCN3-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN3-NEXT: s_lshl_b64 s[0:1], s[14:15], 3 |
| ; GCN3-NEXT: s_add_u32 s0, s8, s0 |
| ; GCN3-NEXT: s_addc_u32 s1, s9, s1 |
| ; GCN3-NEXT: s_cmp_eq_u32 s1, s3 |
| ; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB128_4 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_mov_b64 s[2:3], 0 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s13 |
| ; GCN3-NEXT: v_mov_b32_e32 v5, s12 |
| ; GCN3-NEXT: .LBB128_2: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB128_2 |
| ; GCN3-NEXT: ; %bb.3: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[2:3] |
| ; GCN3-NEXT: s_branch .LBB128_6 |
| ; GCN3-NEXT: .LBB128_4: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_cbranch_execz .LBB128_6 |
| ; GCN3-NEXT: ; %bb.5: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[0:1], 0 |
| ; GCN3-NEXT: s_cselect_b32 s0, s0, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s0 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[16:19], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s12 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s13 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[16:19], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[16:19], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB128_6: ; %atomicrmw.phi |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s10 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s11 |
| ; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1] |
| ; GCN3-NEXT: s_endpgm |
| entry: |
| %ptr = getelementptr i64, ptr %out, i64 %index |
| %tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst |
| store i64 %tmp0, ptr %out2 |
| ret void |
| } |
| |
| define void @flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB129_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB129_6 |
| ; GCN1-NEXT: .LBB129_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB129_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 4, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN1-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB129_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB129_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB129_2 |
| ; GCN1-NEXT: .LBB129_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB129_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB129_6 |
| ; GCN2-NEXT: .LBB129_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB129_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 4, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: flat_load_dword v7, v[4:5] |
| ; GCN2-NEXT: flat_load_dword v6, v[0:1] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB129_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB129_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB129_2 |
| ; GCN2-NEXT: .LBB129_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB129_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB129_6 |
| ; GCN3-NEXT: .LBB129_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB129_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[6:7], v[0:1] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB129_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v7, v5 |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: v_mov_b32_e32 v6, v4 |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB129_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB129_2 |
| ; GCN3-NEXT: .LBB129_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret void |
| } |
| |
| define i64 @flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB130_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB130_6 |
| ; GCN1-NEXT: .LBB130_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB130_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 4, v4 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN1-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN1-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN1-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN1-NEXT: .LBB130_4: ; %atomicrmw.start |
| ; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB130_4 |
| ; GCN1-NEXT: ; %bb.5: ; %Flow |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB130_2 |
| ; GCN1-NEXT: .LBB130_6: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB130_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB130_6 |
| ; GCN2-NEXT: .LBB130_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB130_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 4, v4 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc |
| ; GCN2-NEXT: flat_load_dword v1, v[0:1] |
| ; GCN2-NEXT: flat_load_dword v0, v[4:5] |
| ; GCN2-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN2-NEXT: .LBB130_4: ; %atomicrmw.start |
| ; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB130_4 |
| ; GCN2-NEXT: ; %bb.5: ; %Flow |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB130_2 |
| ; GCN2-NEXT: .LBB130_6: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB130_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB130_6 |
| ; GCN3-NEXT: .LBB130_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB130_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[4:5] |
| ; GCN3-NEXT: s_mov_b64 s[6:7], 0 |
| ; GCN3-NEXT: .LBB130_4: ; %atomicrmw.start |
| ; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v9, v1 |
| ; GCN3-NEXT: v_mov_b32_e32 v8, v0 |
| ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc |
| ; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9] |
| ; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7] |
| ; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB130_4 |
| ; GCN3-NEXT: ; %bb.5: ; %Flow |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB130_2 |
| ; GCN3-NEXT: .LBB130_6: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw min ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret i64 %result |
| } |
| |
| ; --------------------------------------------------------------------- |
| ; atomicrmw uinc_wrap |
| ; --------------------------------------------------------------------- |
| |
| define void @flat_atomic_uinc_wrap_i64_noret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB131_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB131_4 |
| ; GCN1-NEXT: .LBB131_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB131_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB131_2 |
| ; GCN1-NEXT: .LBB131_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB131_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB131_4 |
| ; GCN2-NEXT: .LBB131_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB131_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB131_2 |
| ; GCN2-NEXT: .LBB131_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB131_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB131_4 |
| ; GCN3-NEXT: .LBB131_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB131_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB131_2 |
| ; GCN3-NEXT: .LBB131_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define void @flat_atomic_uinc_wrap_i64_noret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB132_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB132_4 |
| ; GCN1-NEXT: .LBB132_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB132_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB132_2 |
| ; GCN1-NEXT: .LBB132_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB132_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB132_4 |
| ; GCN2-NEXT: .LBB132_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB132_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB132_2 |
| ; GCN2-NEXT: .LBB132_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB132_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB132_4 |
| ; GCN3-NEXT: .LBB132_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB132_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB132_2 |
| ; GCN3-NEXT: .LBB132_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define i64 @flat_atomic_uinc_wrap_i64_ret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB133_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB133_4 |
| ; GCN1-NEXT: .LBB133_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB133_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB133_2 |
| ; GCN1-NEXT: .LBB133_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc |
| ; GCN1-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB133_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB133_4 |
| ; GCN2-NEXT: .LBB133_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB133_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB133_2 |
| ; GCN2-NEXT: .LBB133_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc |
| ; GCN2-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB133_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB133_4 |
| ; GCN3-NEXT: .LBB133_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB133_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB133_2 |
| ; GCN3-NEXT: .LBB133_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, 0, v6, vcc |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define i64 @flat_atomic_uinc_wrap_i64_ret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB134_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB134_4 |
| ; GCN1-NEXT: .LBB134_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB134_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB134_2 |
| ; GCN1-NEXT: .LBB134_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc |
| ; GCN1-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB134_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB134_4 |
| ; GCN2-NEXT: .LBB134_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB134_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB134_2 |
| ; GCN2-NEXT: .LBB134_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc |
| ; GCN2-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB134_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB134_4 |
| ; GCN3-NEXT: .LBB134_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB134_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB134_2 |
| ; GCN3-NEXT: .LBB134_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, 0, v6, vcc |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB135_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB135_4 |
| ; GCN1-NEXT: .LBB135_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB135_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB135_2 |
| ; GCN1-NEXT: .LBB135_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v0, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB135_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB135_4 |
| ; GCN2-NEXT: .LBB135_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB135_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB135_2 |
| ; GCN2-NEXT: .LBB135_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v0, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB135_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB135_4 |
| ; GCN3-NEXT: .LBB135_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB135_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB135_2 |
| ; GCN3-NEXT: .LBB135_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v3, vcc, 1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v4, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB136_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB136_4 |
| ; GCN1-NEXT: .LBB136_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB136_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB136_2 |
| ; GCN1-NEXT: .LBB136_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v0, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB136_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB136_4 |
| ; GCN2-NEXT: .LBB136_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB136_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB136_2 |
| ; GCN2-NEXT: .LBB136_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v5, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v0, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB136_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB136_4 |
| ; GCN3-NEXT: .LBB136_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB136_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB136_2 |
| ; GCN3-NEXT: .LBB136_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v3, vcc, 1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v4, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB137_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB137_3 |
| ; GCN1-NEXT: s_branch .LBB137_4 |
| ; GCN1-NEXT: .LBB137_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB137_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc |
| ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB137_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB137_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB137_3 |
| ; GCN2-NEXT: s_branch .LBB137_4 |
| ; GCN2-NEXT: .LBB137_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB137_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc |
| ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB137_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB137_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB137_3 |
| ; GCN3-NEXT: s_branch .LBB137_4 |
| ; GCN3-NEXT: .LBB137_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB137_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v3, vcc, 1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB137_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB138_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB138_3 |
| ; GCN1-NEXT: s_branch .LBB138_4 |
| ; GCN1-NEXT: .LBB138_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB138_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc |
| ; GCN1-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB138_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB138_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB138_3 |
| ; GCN2-NEXT: s_branch .LBB138_4 |
| ; GCN2-NEXT: .LBB138_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB138_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc |
| ; GCN2-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v5, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB138_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB138_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB138_3 |
| ; GCN3-NEXT: s_branch .LBB138_4 |
| ; GCN3-NEXT: .LBB138_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB138_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v3, vcc, 1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v1, vcc |
| ; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB138_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define void @flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB139_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB139_4 |
| ; GCN1-NEXT: .LBB139_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB139_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB139_2 |
| ; GCN1-NEXT: .LBB139_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc |
| ; GCN1-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB139_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB139_4 |
| ; GCN2-NEXT: .LBB139_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB139_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB139_2 |
| ; GCN2-NEXT: .LBB139_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v6, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v7, vcc |
| ; GCN2-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v0, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB139_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB139_4 |
| ; GCN3-NEXT: .LBB139_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB139_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB139_2 |
| ; GCN3-NEXT: .LBB139_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret void |
| } |
| |
| define i64 @flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB140_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB140_4 |
| ; GCN1-NEXT: .LBB140_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB140_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB140_2 |
| ; GCN1-NEXT: .LBB140_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e32 v6, vcc, 1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc |
| ; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc |
| ; GCN1-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v2, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB140_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB140_4 |
| ; GCN2-NEXT: .LBB140_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB140_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB140_2 |
| ; GCN2-NEXT: .LBB140_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e32 v6, vcc, 1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc |
| ; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, 0, v7, vcc |
| ; GCN2-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v2, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB140_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB140_4 |
| ; GCN3-NEXT: .LBB140_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB140_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB140_2 |
| ; GCN3-NEXT: .LBB140_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e32 v5, vcc, 1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc |
| ; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, 0, v6, vcc |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret i64 %result |
| } |
| |
| ; --------------------------------------------------------------------- |
| ; atomicrmw udec_wrap |
| ; --------------------------------------------------------------------- |
| |
| define void @flat_atomic_udec_wrap_i64_noret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB141_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB141_4 |
| ; GCN1-NEXT: .LBB141_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB141_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB141_2 |
| ; GCN1-NEXT: .LBB141_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN1-NEXT: v_add_i32_e64 v0, s[6:7], -1, v0 |
| ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN1-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB141_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB141_4 |
| ; GCN2-NEXT: .LBB141_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB141_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB141_2 |
| ; GCN2-NEXT: .LBB141_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN2-NEXT: v_add_u32_e64 v0, s[6:7], -1, v0 |
| ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN2-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB141_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB141_4 |
| ; GCN3-NEXT: .LBB141_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB141_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB141_2 |
| ; GCN3-NEXT: .LBB141_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN3-NEXT: v_add_co_u32_e64 v0, s[6:7], -1, v0 |
| ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[6:7], -1, v1, s[6:7] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define void @flat_atomic_udec_wrap_i64_noret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB142_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB142_4 |
| ; GCN1-NEXT: .LBB142_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB142_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB142_2 |
| ; GCN1-NEXT: .LBB142_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN1-NEXT: v_add_i32_e64 v0, s[6:7], -1, v0 |
| ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN1-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB142_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB142_4 |
| ; GCN2-NEXT: .LBB142_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB142_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB142_2 |
| ; GCN2-NEXT: .LBB142_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN2-NEXT: v_add_u32_e64 v0, s[6:7], -1, v0 |
| ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN2-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB142_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB142_4 |
| ; GCN3-NEXT: .LBB142_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB142_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB142_2 |
| ; GCN3-NEXT: .LBB142_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN3-NEXT: v_add_co_u32_e64 v0, s[6:7], -1, v0 |
| ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[6:7], -1, v1, s[6:7] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define i64 @flat_atomic_udec_wrap_i64_ret(ptr %ptr, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN1-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB143_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB143_4 |
| ; GCN1-NEXT: .LBB143_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB143_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB143_2 |
| ; GCN1-NEXT: .LBB143_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e64 v6, s[6:7], -1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN1-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7] |
| ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB143_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB143_4 |
| ; GCN2-NEXT: .LBB143_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB143_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB143_2 |
| ; GCN2-NEXT: .LBB143_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e64 v6, s[6:7], -1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN2-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7] |
| ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_mov_b32_e32 v5, v1 |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_mov_b32_e32 v4, v0 |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB143_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB143_4 |
| ; GCN3-NEXT: .LBB143_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB143_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB143_2 |
| ; GCN3-NEXT: .LBB143_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e64 v5, s[6:7], -1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN3-NEXT: v_addc_co_u32_e64 v6, s[6:7], -1, v1, s[6:7] |
| ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define i64 @flat_atomic_udec_wrap_i64_ret_offset(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB144_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB144_4 |
| ; GCN1-NEXT: .LBB144_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB144_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB144_2 |
| ; GCN1-NEXT: .LBB144_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e64 v6, s[6:7], -1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN1-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7] |
| ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB144_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB144_4 |
| ; GCN2-NEXT: .LBB144_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB144_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB144_2 |
| ; GCN2-NEXT: .LBB144_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e64 v6, s[6:7], -1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN2-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7] |
| ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB144_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB144_4 |
| ; GCN3-NEXT: .LBB144_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB144_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB144_2 |
| ; GCN3-NEXT: .LBB144_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e64 v5, s[6:7], -1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN3-NEXT: v_addc_co_u32_e64 v6, s[6:7], -1, v1, s[6:7] |
| ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB145_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB145_4 |
| ; GCN1-NEXT: .LBB145_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB145_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB145_2 |
| ; GCN1-NEXT: .LBB145_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] |
| ; GCN1-NEXT: v_add_i32_e64 v0, s[36:37], -1, v0 |
| ; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35] |
| ; GCN1-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v1, s[36:37] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB145_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB145_4 |
| ; GCN2-NEXT: .LBB145_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB145_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB145_2 |
| ; GCN2-NEXT: .LBB145_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] |
| ; GCN2-NEXT: v_add_u32_e64 v0, s[36:37], -1, v0 |
| ; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35] |
| ; GCN2-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v1, s[36:37] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_mov_b64 s[34:35], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB145_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB145_4 |
| ; GCN3-NEXT: .LBB145_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB145_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB145_2 |
| ; GCN3-NEXT: .LBB145_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] |
| ; GCN3-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v0 |
| ; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35] |
| ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v1, s[36:37] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN1-NEXT: s_cbranch_vccnz .LBB146_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB146_4 |
| ; GCN1-NEXT: .LBB146_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB146_3: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execnz .LBB146_2 |
| ; GCN1-NEXT: .LBB146_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] |
| ; GCN1-NEXT: v_add_i32_e64 v0, s[36:37], -1, v0 |
| ; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35] |
| ; GCN1-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v1, s[36:37] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN2-NEXT: s_cbranch_vccnz .LBB146_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB146_4 |
| ; GCN2-NEXT: .LBB146_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB146_3: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execnz .LBB146_2 |
| ; GCN2-NEXT: .LBB146_4: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] |
| ; GCN2-NEXT: v_add_u32_e64 v0, s[36:37], -1, v0 |
| ; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35] |
| ; GCN2-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v1, s[36:37] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_mov_b64 s[36:37], -1 |
| ; GCN3-NEXT: s_cbranch_vccnz .LBB146_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB146_4 |
| ; GCN3-NEXT: .LBB146_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB146_3: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execnz .LBB146_2 |
| ; GCN3-NEXT: .LBB146_4: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] |
| ; GCN3-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v0 |
| ; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35] |
| ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v1, s[36:37] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB147_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB147_3 |
| ; GCN1-NEXT: s_branch .LBB147_4 |
| ; GCN1-NEXT: .LBB147_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB147_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[34:35], s[4:5], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e64 v6, s[36:37], -1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] |
| ; GCN1-NEXT: v_addc_u32_e64 v7, s[36:37], -1, v1, s[36:37] |
| ; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB147_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s34, s[34:35], 0x0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s5, s34 |
| ; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB147_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB147_3 |
| ; GCN2-NEXT: s_branch .LBB147_4 |
| ; GCN2-NEXT: .LBB147_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB147_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e64 v6, s[36:37], -1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] |
| ; GCN2-NEXT: v_addc_u32_e64 v7, s[36:37], -1, v1, s[36:37] |
| ; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB147_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s5, s35 |
| ; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB147_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s4 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s5 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB147_3 |
| ; GCN3-NEXT: s_branch .LBB147_4 |
| ; GCN3-NEXT: .LBB147_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB147_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[4:5], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s4, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e64 v5, s[36:37], -1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] |
| ; GCN3-NEXT: v_addc_co_u32_e64 v6, s[36:37], -1, v1, s[36:37] |
| ; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB147_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) { |
| ; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN1-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN1-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN1-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN1-NEXT: s_cbranch_vccz .LBB148_2 |
| ; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN1-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN1-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: s_cbranch_execz .LBB148_3 |
| ; GCN1-NEXT: s_branch .LBB148_4 |
| ; GCN1-NEXT: .LBB148_2: |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: .LBB148_3: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e64 s[36:37], s[34:35], 0 |
| ; GCN1-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec |
| ; GCN1-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN1-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN1-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN1-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN1-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e64 v6, s[36:37], -1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] |
| ; GCN1-NEXT: v_addc_u32_e64 v7, s[36:37], -1, v1, s[36:37] |
| ; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc |
| ; GCN1-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN1-NEXT: .LBB148_4: ; %atomicrmw.end |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[34:35], 0xe4 |
| ; GCN2-NEXT: s_load_dword s36, s[34:35], 0x0 |
| ; GCN2-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN2-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: s_cmp_eq_u32 s35, s36 |
| ; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN2-NEXT: s_cbranch_vccz .LBB148_2 |
| ; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN2-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: s_cbranch_execz .LBB148_3 |
| ; GCN2-NEXT: s_branch .LBB148_4 |
| ; GCN2-NEXT: .LBB148_2: |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: .LBB148_3: ; %atomicrmw.private |
| ; GCN2-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN2-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN2-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN2-NEXT: s_add_i32 s34, s34, 4 |
| ; GCN2-NEXT: v_mov_b32_e32 v3, s34 |
| ; GCN2-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: v_mov_b32_e32 v5, s6 |
| ; GCN2-NEXT: v_mov_b32_e32 v4, s7 |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e64 v6, s[36:37], -1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] |
| ; GCN2-NEXT: v_addc_u32_e64 v7, s[36:37], -1, v1, s[36:37] |
| ; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc |
| ; GCN2-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen |
| ; GCN2-NEXT: .LBB148_4: ; %atomicrmw.end |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: s_add_u32 s34, s4, 32 |
| ; GCN3-NEXT: s_addc_u32 s35, s5, 0 |
| ; GCN3-NEXT: s_mov_b64 s[36:37], src_private_base |
| ; GCN3-NEXT: s_cmp_eq_u32 s35, s37 |
| ; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0 |
| ; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37] |
| ; GCN3-NEXT: s_cbranch_vccz .LBB148_2 |
| ; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global |
| ; GCN3-NEXT: v_mov_b32_e32 v0, s34 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v1, s35 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: s_cbranch_execz .LBB148_3 |
| ; GCN3-NEXT: s_branch .LBB148_4 |
| ; GCN3-NEXT: .LBB148_2: |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: .LBB148_3: ; %atomicrmw.private |
| ; GCN3-NEXT: s_cmp_lg_u64 s[34:35], 0 |
| ; GCN3-NEXT: s_cselect_b32 s34, s34, -1 |
| ; GCN3-NEXT: v_mov_b32_e32 v2, s34 |
| ; GCN3-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: v_mov_b32_e32 v4, s6 |
| ; GCN3-NEXT: v_mov_b32_e32 v3, s7 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e64 v5, s[36:37], -1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[0:1] |
| ; GCN3-NEXT: v_addc_co_u32_e64 v6, s[36:37], -1, v1, s[36:37] |
| ; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: .LBB148_4: ; %atomicrmw.end |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst |
| ret i64 %result |
| } |
| |
| define void @flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v0, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB149_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB149_4 |
| ; GCN1-NEXT: .LBB149_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB149_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB149_2 |
| ; GCN1-NEXT: .LBB149_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN1-NEXT: v_add_i32_e64 v0, s[6:7], -1, v0 |
| ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN1-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v0, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB149_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB149_4 |
| ; GCN2-NEXT: .LBB149_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB149_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB149_2 |
| ; GCN2-NEXT: .LBB149_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN2-NEXT: v_add_u32_e64 v0, s[6:7], -1, v0 |
| ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN2-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v1, s[6:7] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB149_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB149_4 |
| ; GCN3-NEXT: .LBB149_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB149_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB149_2 |
| ; GCN3-NEXT: .LBB149_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN3-NEXT: v_add_co_u32_e64 v0, s[6:7], -1, v0 |
| ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN3-NEXT: v_addc_co_u32_e64 v1, s[6:7], -1, v1, s[6:7] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret void |
| } |
| |
| define i64 @flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) { |
| ; GCN1-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN1: ; %bb.0: |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN1-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN1-NEXT: v_add_i32_e32 v4, vcc, 32, v0 |
| ; GCN1-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN1-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN1-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN1-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN1-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB150_3 |
| ; GCN1-NEXT: ; %bb.1: ; %Flow |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execnz .LBB150_4 |
| ; GCN1-NEXT: .LBB150_2: ; %atomicrmw.phi |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; GCN1-NEXT: .LBB150_3: ; %atomicrmw.global |
| ; GCN1-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN1-NEXT: buffer_wbinvl1_vol |
| ; GCN1-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN1-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN1-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN1-NEXT: s_cbranch_execz .LBB150_2 |
| ; GCN1-NEXT: .LBB150_4: ; %atomicrmw.private |
| ; GCN1-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN1-NEXT: v_add_i32_e32 v5, vcc, 4, v4 |
| ; GCN1-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_waitcnt vmcnt(1) |
| ; GCN1-NEXT: v_add_i32_e64 v6, s[6:7], -1, v0 |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN1-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN1-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7] |
| ; GCN1-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN1-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc |
| ; GCN1-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc |
| ; GCN1-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN1-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN1-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN1-NEXT: s_waitcnt vmcnt(0) |
| ; GCN1-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN2-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN2: ; %bb.0: |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: s_mov_b64 s[4:5], 0xe4 |
| ; GCN2-NEXT: s_load_dword s4, s[4:5], 0x0 |
| ; GCN2-NEXT: v_add_u32_e32 v4, vcc, 32, v0 |
| ; GCN2-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN2-NEXT: s_waitcnt lgkmcnt(0) |
| ; GCN2-NEXT: v_cmp_ne_u32_e32 vcc, s4, v5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN2-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN2-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB150_3 |
| ; GCN2-NEXT: ; %bb.1: ; %Flow |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execnz .LBB150_4 |
| ; GCN2-NEXT: .LBB150_2: ; %atomicrmw.phi |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; GCN2-NEXT: .LBB150_3: ; %atomicrmw.global |
| ; GCN2-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN2-NEXT: buffer_wbinvl1_vol |
| ; GCN2-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN2-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN2-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN2-NEXT: s_cbranch_execz .LBB150_2 |
| ; GCN2-NEXT: .LBB150_4: ; %atomicrmw.private |
| ; GCN2-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN2-NEXT: v_add_u32_e32 v5, vcc, 4, v4 |
| ; GCN2-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_waitcnt vmcnt(1) |
| ; GCN2-NEXT: v_add_u32_e64 v6, s[6:7], -1, v0 |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN2-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN2-NEXT: v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7] |
| ; GCN2-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN2-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc |
| ; GCN2-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc |
| ; GCN2-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN2-NEXT: buffer_store_dword v3, v5, s[0:3], 0 offen |
| ; GCN2-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN2-NEXT: s_waitcnt vmcnt(0) |
| ; GCN2-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory: |
| ; GCN3: ; %bb.0: |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: v_add_co_u32_e32 v4, vcc, 32, v0 |
| ; GCN3-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc |
| ; GCN3-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GCN3-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GCN3-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GCN3-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB150_3 |
| ; GCN3-NEXT: ; %bb.1: ; %Flow |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execnz .LBB150_4 |
| ; GCN3-NEXT: .LBB150_2: ; %atomicrmw.phi |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| ; GCN3-NEXT: .LBB150_3: ; %atomicrmw.global |
| ; GCN3-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GCN3-NEXT: buffer_wbinvl1_vol |
| ; GCN3-NEXT: ; implicit-def: $vgpr4_vgpr5 |
| ; GCN3-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GCN3-NEXT: s_andn2_saveexec_b64 s[8:9], s[4:5] |
| ; GCN3-NEXT: s_cbranch_execz .LBB150_2 |
| ; GCN3-NEXT: .LBB150_4: ; %atomicrmw.private |
| ; GCN3-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc |
| ; GCN3-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_waitcnt vmcnt(1) |
| ; GCN3-NEXT: v_add_co_u32_e64 v5, s[6:7], -1, v0 |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1] |
| ; GCN3-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3] |
| ; GCN3-NEXT: v_addc_co_u32_e64 v6, s[6:7], -1, v1, s[6:7] |
| ; GCN3-NEXT: s_or_b64 vcc, vcc, s[4:5] |
| ; GCN3-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc |
| ; GCN3-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc |
| ; GCN3-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen |
| ; GCN3-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:4 |
| ; GCN3-NEXT: s_or_b64 exec, exec, s[8:9] |
| ; GCN3-NEXT: s_waitcnt vmcnt(0) |
| ; GCN3-NEXT: s_setpc_b64 s[30:31] |
| %gep = getelementptr i64, ptr %out, i64 4 |
| %result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0 |
| ret i64 %result |
| } |
| |
| !0 = !{} |
