| ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py |
| ; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX908 %s |
| ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefix=GFX90A %s |
| ; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX942 %s |
| ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX1100 %s |
| ; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX1200 %s |
| |
| define float @syncscope_system(ptr %addr, float %val) #0 { |
| ; GFX908-LABEL: syncscope_system: |
| ; GFX908: ; %bb.0: |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: flat_load_dword v3, v[0:1] |
| ; GFX908-NEXT: s_mov_b64 s[4:5], 0 |
| ; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start |
| ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: v_mov_b32_e32 v4, v3 |
| ; GFX908-NEXT: v_add_f32_e32 v3, v4, v2 |
| ; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: buffer_wbinvl1_vol |
| ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4 |
| ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GFX908-NEXT: s_cbranch_execnz .LBB0_1 |
| ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end |
| ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GFX908-NEXT: v_mov_b32_e32 v0, v3 |
| ; GFX908-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX90A-LABEL: syncscope_system: |
| ; GFX90A: ; %bb.0: |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX90A-NEXT: s_mov_b64 s[4:5], src_shared_base |
| ; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GFX90A-NEXT: ; implicit-def: $vgpr3 |
| ; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GFX90A-NEXT: s_cbranch_execz .LBB0_6 |
| ; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private |
| ; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base |
| ; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s7, v1 |
| ; GFX90A-NEXT: ; implicit-def: $vgpr3 |
| ; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc |
| ; GFX90A-NEXT: s_xor_b64 s[6:7], exec, s[6:7] |
| ; GFX90A-NEXT: s_cbranch_execz .LBB0_3 |
| ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global |
| ; GFX90A-NEXT: buffer_wbl2 |
| ; GFX90A-NEXT: global_atomic_add_f32 v3, v[0:1], v2, off glc |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) |
| ; GFX90A-NEXT: buffer_invl2 |
| ; GFX90A-NEXT: buffer_wbinvl1_vol |
| ; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GFX90A-NEXT: ; implicit-def: $vgpr2 |
| ; GFX90A-NEXT: .LBB0_3: ; %Flow |
| ; GFX90A-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7] |
| ; GFX90A-NEXT: s_cbranch_execz .LBB0_5 |
| ; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private |
| ; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) |
| ; GFX90A-NEXT: v_add_f32_e32 v1, v3, v2 |
| ; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GFX90A-NEXT: .LBB0_5: ; %Flow1 |
| ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GFX90A-NEXT: ; implicit-def: $vgpr2 |
| ; GFX90A-NEXT: .LBB0_6: ; %Flow2 |
| ; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GFX90A-NEXT: s_cbranch_execz .LBB0_8 |
| ; GFX90A-NEXT: ; %bb.7: ; %atomicrmw.shared |
| ; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GFX90A-NEXT: ds_add_rtn_f32 v3, v0, v2 |
| ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) |
| ; GFX90A-NEXT: .LBB0_8: ; %atomicrmw.phi |
| ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GFX90A-NEXT: v_mov_b32_e32 v0, v3 |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) |
| ; GFX90A-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX942-LABEL: syncscope_system: |
| ; GFX942: ; %bb.0: |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX942-NEXT: buffer_wbl2 sc0 sc1 |
| ; GFX942-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 sc0 sc1 |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX942-NEXT: buffer_inv sc0 sc1 |
| ; GFX942-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX1100-LABEL: syncscope_system: |
| ; GFX1100: ; %bb.0: |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX1100-NEXT: s_waitcnt_vscnt null, 0x0 |
| ; GFX1100-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 glc |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX1100-NEXT: buffer_gl1_inv |
| ; GFX1100-NEXT: buffer_gl0_inv |
| ; GFX1100-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX1200-LABEL: syncscope_system: |
| ; GFX1200: ; %bb.0: |
| ; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0 |
| ; GFX1200-NEXT: s_wait_expcnt 0x0 |
| ; GFX1200-NEXT: s_wait_samplecnt 0x0 |
| ; GFX1200-NEXT: s_wait_bvhcnt 0x0 |
| ; GFX1200-NEXT: s_wait_kmcnt 0x0 |
| ; GFX1200-NEXT: global_wb scope:SCOPE_SYS |
| ; GFX1200-NEXT: s_wait_storecnt 0x0 |
| ; GFX1200-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS |
| ; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0 |
| ; GFX1200-NEXT: global_inv scope:SCOPE_SYS |
| ; GFX1200-NEXT: s_setpc_b64 s[30:31] |
| %res = atomicrmw fadd ptr %addr, float %val seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0 |
| ret float %res |
| } |
| |
| define float @syncscope_workgroup_rtn(ptr %addr, float %val) #0 { |
| ; GFX908-LABEL: syncscope_workgroup_rtn: |
| ; GFX908: ; %bb.0: |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: flat_load_dword v3, v[0:1] |
| ; GFX908-NEXT: s_mov_b64 s[4:5], 0 |
| ; GFX908-NEXT: .LBB1_1: ; %atomicrmw.start |
| ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: v_mov_b32_e32 v4, v3 |
| ; GFX908-NEXT: v_add_f32_e32 v3, v4, v2 |
| ; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4 |
| ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GFX908-NEXT: s_cbranch_execnz .LBB1_1 |
| ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end |
| ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GFX908-NEXT: v_mov_b32_e32 v0, v3 |
| ; GFX908-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX90A-LABEL: syncscope_workgroup_rtn: |
| ; GFX90A: ; %bb.0: |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX90A-NEXT: s_mov_b64 s[4:5], src_shared_base |
| ; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GFX90A-NEXT: ; implicit-def: $vgpr3 |
| ; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GFX90A-NEXT: s_cbranch_execz .LBB1_6 |
| ; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private |
| ; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base |
| ; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s7, v1 |
| ; GFX90A-NEXT: ; implicit-def: $vgpr3 |
| ; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc |
| ; GFX90A-NEXT: s_xor_b64 s[6:7], exec, s[6:7] |
| ; GFX90A-NEXT: s_cbranch_execz .LBB1_3 |
| ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global |
| ; GFX90A-NEXT: global_atomic_add_f32 v3, v[0:1], v2, off glc |
| ; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GFX90A-NEXT: ; implicit-def: $vgpr2 |
| ; GFX90A-NEXT: .LBB1_3: ; %Flow |
| ; GFX90A-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7] |
| ; GFX90A-NEXT: s_cbranch_execz .LBB1_5 |
| ; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private |
| ; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) |
| ; GFX90A-NEXT: v_add_f32_e32 v1, v3, v2 |
| ; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GFX90A-NEXT: .LBB1_5: ; %Flow1 |
| ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GFX90A-NEXT: ; implicit-def: $vgpr2 |
| ; GFX90A-NEXT: .LBB1_6: ; %Flow2 |
| ; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GFX90A-NEXT: s_cbranch_execz .LBB1_8 |
| ; GFX90A-NEXT: ; %bb.7: ; %atomicrmw.shared |
| ; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) |
| ; GFX90A-NEXT: ds_add_rtn_f32 v3, v0, v2 |
| ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) |
| ; GFX90A-NEXT: .LBB1_8: ; %atomicrmw.phi |
| ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) |
| ; GFX90A-NEXT: v_mov_b32_e32 v0, v3 |
| ; GFX90A-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX942-LABEL: syncscope_workgroup_rtn: |
| ; GFX942: ; %bb.0: |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX942-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 sc0 |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX942-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX1100-LABEL: syncscope_workgroup_rtn: |
| ; GFX1100: ; %bb.0: |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX1100-NEXT: s_waitcnt_vscnt null, 0x0 |
| ; GFX1100-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 glc |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX1100-NEXT: buffer_gl0_inv |
| ; GFX1100-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX1200-LABEL: syncscope_workgroup_rtn: |
| ; GFX1200: ; %bb.0: |
| ; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0 |
| ; GFX1200-NEXT: s_wait_expcnt 0x0 |
| ; GFX1200-NEXT: s_wait_samplecnt 0x0 |
| ; GFX1200-NEXT: s_wait_bvhcnt 0x0 |
| ; GFX1200-NEXT: s_wait_kmcnt 0x0 |
| ; GFX1200-NEXT: s_wait_storecnt 0x0 |
| ; GFX1200-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SE |
| ; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0 |
| ; GFX1200-NEXT: global_inv scope:SCOPE_SE |
| ; GFX1200-NEXT: s_setpc_b64 s[30:31] |
| %res = atomicrmw fadd ptr %addr, float %val syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0 |
| ret float %res |
| } |
| |
| define void @syncscope_workgroup_nortn(ptr %addr, float %val) #0 { |
| ; GFX908-LABEL: syncscope_workgroup_nortn: |
| ; GFX908: ; %bb.0: |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: s_mov_b64 s[4:5], src_shared_base |
| ; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GFX908-NEXT: s_cbranch_execnz .LBB2_3 |
| ; GFX908-NEXT: ; %bb.1: ; %Flow2 |
| ; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GFX908-NEXT: s_cbranch_execnz .LBB2_8 |
| ; GFX908-NEXT: .LBB2_2: ; %atomicrmw.phi |
| ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) |
| ; GFX908-NEXT: s_setpc_b64 s[30:31] |
| ; GFX908-NEXT: .LBB2_3: ; %atomicrmw.check.private |
| ; GFX908-NEXT: s_mov_b64 s[6:7], src_private_base |
| ; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s7, v1 |
| ; GFX908-NEXT: s_and_saveexec_b64 s[6:7], vcc |
| ; GFX908-NEXT: s_xor_b64 s[6:7], exec, s[6:7] |
| ; GFX908-NEXT: s_cbranch_execz .LBB2_5 |
| ; GFX908-NEXT: ; %bb.4: ; %atomicrmw.global |
| ; GFX908-NEXT: global_atomic_add_f32 v[0:1], v2, off |
| ; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GFX908-NEXT: ; implicit-def: $vgpr2 |
| ; GFX908-NEXT: .LBB2_5: ; %Flow |
| ; GFX908-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7] |
| ; GFX908-NEXT: s_cbranch_execz .LBB2_7 |
| ; GFX908-NEXT: ; %bb.6: ; %atomicrmw.private |
| ; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GFX908-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GFX908-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) |
| ; GFX908-NEXT: v_add_f32_e32 v1, v1, v2 |
| ; GFX908-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GFX908-NEXT: .LBB2_7: ; %Flow1 |
| ; GFX908-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GFX908-NEXT: ; implicit-def: $vgpr2 |
| ; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GFX908-NEXT: s_cbranch_execz .LBB2_2 |
| ; GFX908-NEXT: .LBB2_8: ; %atomicrmw.shared |
| ; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GFX908-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GFX908-NEXT: ds_add_f32 v0, v2 |
| ; GFX908-NEXT: s_waitcnt lgkmcnt(0) |
| ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) |
| ; GFX908-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX90A-LABEL: syncscope_workgroup_nortn: |
| ; GFX90A: ; %bb.0: |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX90A-NEXT: s_mov_b64 s[4:5], src_shared_base |
| ; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1 |
| ; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc |
| ; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5] |
| ; GFX90A-NEXT: s_cbranch_execnz .LBB2_3 |
| ; GFX90A-NEXT: ; %bb.1: ; %Flow2 |
| ; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GFX90A-NEXT: s_cbranch_execnz .LBB2_8 |
| ; GFX90A-NEXT: .LBB2_2: ; %atomicrmw.phi |
| ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) |
| ; GFX90A-NEXT: s_setpc_b64 s[30:31] |
| ; GFX90A-NEXT: .LBB2_3: ; %atomicrmw.check.private |
| ; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base |
| ; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s7, v1 |
| ; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc |
| ; GFX90A-NEXT: s_xor_b64 s[6:7], exec, s[6:7] |
| ; GFX90A-NEXT: s_cbranch_execz .LBB2_5 |
| ; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.global |
| ; GFX90A-NEXT: global_atomic_add_f32 v[0:1], v2, off |
| ; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GFX90A-NEXT: ; implicit-def: $vgpr2 |
| ; GFX90A-NEXT: .LBB2_5: ; %Flow |
| ; GFX90A-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7] |
| ; GFX90A-NEXT: s_cbranch_execz .LBB2_7 |
| ; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.private |
| ; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) |
| ; GFX90A-NEXT: v_add_f32_e32 v1, v1, v2 |
| ; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GFX90A-NEXT: .LBB2_7: ; %Flow1 |
| ; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7] |
| ; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1 |
| ; GFX90A-NEXT: ; implicit-def: $vgpr2 |
| ; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5] |
| ; GFX90A-NEXT: s_cbranch_execz .LBB2_2 |
| ; GFX90A-NEXT: .LBB2_8: ; %atomicrmw.shared |
| ; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1] |
| ; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc |
| ; GFX90A-NEXT: ds_add_f32 v0, v2 |
| ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) |
| ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) |
| ; GFX90A-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX942-LABEL: syncscope_workgroup_nortn: |
| ; GFX942: ; %bb.0: |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX942-NEXT: flat_atomic_add_f32 v[0:1], v2 |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX942-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX1100-LABEL: syncscope_workgroup_nortn: |
| ; GFX1100: ; %bb.0: |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX1100-NEXT: s_waitcnt_vscnt null, 0x0 |
| ; GFX1100-NEXT: flat_atomic_add_f32 v[0:1], v2 |
| ; GFX1100-NEXT: s_waitcnt lgkmcnt(0) |
| ; GFX1100-NEXT: s_waitcnt_vscnt null, 0x0 |
| ; GFX1100-NEXT: buffer_gl0_inv |
| ; GFX1100-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX1200-LABEL: syncscope_workgroup_nortn: |
| ; GFX1200: ; %bb.0: |
| ; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0 |
| ; GFX1200-NEXT: s_wait_expcnt 0x0 |
| ; GFX1200-NEXT: s_wait_samplecnt 0x0 |
| ; GFX1200-NEXT: s_wait_bvhcnt 0x0 |
| ; GFX1200-NEXT: s_wait_kmcnt 0x0 |
| ; GFX1200-NEXT: s_wait_storecnt 0x0 |
| ; GFX1200-NEXT: flat_atomic_add_f32 v[0:1], v2 scope:SCOPE_SE |
| ; GFX1200-NEXT: s_wait_storecnt_dscnt 0x0 |
| ; GFX1200-NEXT: global_inv scope:SCOPE_SE |
| ; GFX1200-NEXT: s_setpc_b64 s[30:31] |
| %res = atomicrmw fadd ptr %addr, float %val syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0 |
| ret void |
| } |
| |
| define float @no_unsafe(ptr %addr, float %val) { |
| ; GFX908-LABEL: no_unsafe: |
| ; GFX908: ; %bb.0: |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: flat_load_dword v3, v[0:1] |
| ; GFX908-NEXT: s_mov_b64 s[4:5], 0 |
| ; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start |
| ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: v_mov_b32_e32 v4, v3 |
| ; GFX908-NEXT: v_add_f32_e32 v3, v4, v2 |
| ; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4 |
| ; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GFX908-NEXT: s_cbranch_execnz .LBB3_1 |
| ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end |
| ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GFX908-NEXT: v_mov_b32_e32 v0, v3 |
| ; GFX908-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX90A-LABEL: no_unsafe: |
| ; GFX90A: ; %bb.0: |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX90A-NEXT: flat_load_dword v3, v[0:1] |
| ; GFX90A-NEXT: s_mov_b64 s[4:5], 0 |
| ; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start |
| ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX90A-NEXT: v_mov_b32_e32 v5, v3 |
| ; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2 |
| ; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5 |
| ; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5] |
| ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GFX90A-NEXT: s_cbranch_execnz .LBB3_1 |
| ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end |
| ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GFX90A-NEXT: v_mov_b32_e32 v0, v3 |
| ; GFX90A-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX942-LABEL: no_unsafe: |
| ; GFX942: ; %bb.0: |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX942-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 sc0 |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX942-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX1100-LABEL: no_unsafe: |
| ; GFX1100: ; %bb.0: |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX1100-NEXT: flat_load_b32 v3, v[0:1] |
| ; GFX1100-NEXT: s_mov_b32 s0, 0 |
| ; GFX1100-NEXT: .LBB3_1: ; %atomicrmw.start |
| ; GFX1100-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX1100-NEXT: v_mov_b32_e32 v4, v3 |
| ; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) |
| ; GFX1100-NEXT: v_add_f32_e32 v3, v4, v2 |
| ; GFX1100-NEXT: s_waitcnt_vscnt null, 0x0 |
| ; GFX1100-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX1100-NEXT: buffer_gl0_inv |
| ; GFX1100-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4 |
| ; GFX1100-NEXT: s_or_b32 s0, vcc_lo, s0 |
| ; GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1) |
| ; GFX1100-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 |
| ; GFX1100-NEXT: s_cbranch_execnz .LBB3_1 |
| ; GFX1100-NEXT: ; %bb.2: ; %atomicrmw.end |
| ; GFX1100-NEXT: s_or_b32 exec_lo, exec_lo, s0 |
| ; GFX1100-NEXT: v_mov_b32_e32 v0, v3 |
| ; GFX1100-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX1200-LABEL: no_unsafe: |
| ; GFX1200: ; %bb.0: |
| ; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0 |
| ; GFX1200-NEXT: s_wait_expcnt 0x0 |
| ; GFX1200-NEXT: s_wait_samplecnt 0x0 |
| ; GFX1200-NEXT: s_wait_bvhcnt 0x0 |
| ; GFX1200-NEXT: s_wait_kmcnt 0x0 |
| ; GFX1200-NEXT: s_wait_storecnt 0x0 |
| ; GFX1200-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SE |
| ; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0 |
| ; GFX1200-NEXT: global_inv scope:SCOPE_SE |
| ; GFX1200-NEXT: s_setpc_b64 s[30:31] |
| %res = atomicrmw fadd ptr %addr, float %val syncscope("workgroup") seq_cst |
| ret float %res |
| } |
| |
| @global = hidden addrspace(1) global i64 0, align 8 |
| |
| ; Make sure there is no error on an invalid addrspacecast without optimizations |
| define i64 @optnone_atomicrmw_add_i64_expand(i64 %val) #1 { |
| ; GFX908-LABEL: optnone_atomicrmw_add_i64_expand: |
| ; GFX908: ; %bb.0: |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GFX908-NEXT: s_mov_b32 s6, 32 |
| ; GFX908-NEXT: s_lshr_b64 s[4:5], s[4:5], s6 |
| ; GFX908-NEXT: s_getpc_b64 s[6:7] |
| ; GFX908-NEXT: s_add_u32 s6, s6, global@rel32@lo+4 |
| ; GFX908-NEXT: s_addc_u32 s7, s7, global@rel32@hi+12 |
| ; GFX908-NEXT: s_cmp_eq_u32 s7, s4 |
| ; GFX908-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GFX908-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5] |
| ; GFX908-NEXT: s_mov_b64 s[4:5], -1 |
| ; GFX908-NEXT: s_mov_b32 s6, 1 |
| ; GFX908-NEXT: v_cmp_ne_u32_e64 s[6:7], v2, s6 |
| ; GFX908-NEXT: s_and_b64 vcc, exec, s[6:7] |
| ; GFX908-NEXT: ; implicit-def: $vgpr3_vgpr4 |
| ; GFX908-NEXT: s_cbranch_vccnz .LBB4_3 |
| ; GFX908-NEXT: .LBB4_1: ; %Flow |
| ; GFX908-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5] |
| ; GFX908-NEXT: s_mov_b32 s4, 1 |
| ; GFX908-NEXT: v_cmp_ne_u32_e64 s[4:5], v2, s4 |
| ; GFX908-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GFX908-NEXT: s_cbranch_vccnz .LBB4_4 |
| ; GFX908-NEXT: ; %bb.2: ; %atomicrmw.private |
| ; GFX908-NEXT: s_waitcnt lgkmcnt(0) |
| ; GFX908-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) |
| ; GFX908-NEXT: v_mov_b32_e32 v4, v3 |
| ; GFX908-NEXT: v_add_co_u32_e64 v0, s[4:5], v3, v0 |
| ; GFX908-NEXT: v_addc_co_u32_e64 v1, s[4:5], v4, v1, s[4:5] |
| ; GFX908-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GFX908-NEXT: buffer_store_dword v0, v0, s[0:3], 0 offen |
| ; GFX908-NEXT: s_branch .LBB4_4 |
| ; GFX908-NEXT: .LBB4_3: ; %atomicrmw.global |
| ; GFX908-NEXT: s_getpc_b64 s[4:5] |
| ; GFX908-NEXT: s_add_u32 s4, s4, global@rel32@lo+4 |
| ; GFX908-NEXT: s_addc_u32 s5, s5, global@rel32@hi+12 |
| ; GFX908-NEXT: v_mov_b32_e32 v2, s4 |
| ; GFX908-NEXT: v_mov_b32_e32 v3, s5 |
| ; GFX908-NEXT: flat_atomic_add_x2 v[3:4], v[2:3], v[0:1] glc |
| ; GFX908-NEXT: s_mov_b64 s[4:5], 0 |
| ; GFX908-NEXT: s_branch .LBB4_1 |
| ; GFX908-NEXT: .LBB4_4: ; %atomicrmw.phi |
| ; GFX908-NEXT: ; %bb.5: ; %atomicrmw.end |
| ; GFX908-NEXT: s_mov_b32 s4, 32 |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: v_lshrrev_b64 v[1:2], s4, v[3:4] |
| ; GFX908-NEXT: v_mov_b32_e32 v0, v3 |
| ; GFX908-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX90A-LABEL: optnone_atomicrmw_add_i64_expand: |
| ; GFX90A: ; %bb.0: |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GFX90A-NEXT: s_mov_b32 s6, 32 |
| ; GFX90A-NEXT: s_lshr_b64 s[4:5], s[4:5], s6 |
| ; GFX90A-NEXT: s_getpc_b64 s[6:7] |
| ; GFX90A-NEXT: s_add_u32 s6, s6, global@rel32@lo+4 |
| ; GFX90A-NEXT: s_addc_u32 s7, s7, global@rel32@hi+12 |
| ; GFX90A-NEXT: s_cmp_eq_u32 s7, s4 |
| ; GFX90A-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GFX90A-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5] |
| ; GFX90A-NEXT: s_mov_b64 s[4:5], -1 |
| ; GFX90A-NEXT: s_mov_b32 s6, 1 |
| ; GFX90A-NEXT: v_cmp_ne_u32_e64 s[6:7], v2, s6 |
| ; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7] |
| ; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GFX90A-NEXT: s_cbranch_vccnz .LBB4_3 |
| ; GFX90A-NEXT: .LBB4_1: ; %Flow |
| ; GFX90A-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5] |
| ; GFX90A-NEXT: s_mov_b32 s4, 1 |
| ; GFX90A-NEXT: v_cmp_ne_u32_e64 s[4:5], v4, s4 |
| ; GFX90A-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GFX90A-NEXT: s_cbranch_vccnz .LBB4_4 |
| ; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.private |
| ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) |
| ; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) |
| ; GFX90A-NEXT: v_mov_b32_e32 v3, v2 |
| ; GFX90A-NEXT: v_add_co_u32_e64 v0, s[4:5], v2, v0 |
| ; GFX90A-NEXT: v_addc_co_u32_e64 v1, s[4:5], v3, v1, s[4:5] |
| ; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GFX90A-NEXT: buffer_store_dword v0, v0, s[0:3], 0 offen |
| ; GFX90A-NEXT: s_branch .LBB4_4 |
| ; GFX90A-NEXT: .LBB4_3: ; %atomicrmw.global |
| ; GFX90A-NEXT: s_getpc_b64 s[4:5] |
| ; GFX90A-NEXT: s_add_u32 s4, s4, global@rel32@lo+4 |
| ; GFX90A-NEXT: s_addc_u32 s5, s5, global@rel32@hi+12 |
| ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] |
| ; GFX90A-NEXT: flat_atomic_add_x2 v[2:3], v[2:3], v[0:1] glc |
| ; GFX90A-NEXT: s_mov_b64 s[4:5], 0 |
| ; GFX90A-NEXT: s_branch .LBB4_1 |
| ; GFX90A-NEXT: .LBB4_4: ; %atomicrmw.phi |
| ; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.end |
| ; GFX90A-NEXT: s_mov_b32 s4, 32 |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX90A-NEXT: v_lshrrev_b64 v[4:5], s4, v[2:3] |
| ; GFX90A-NEXT: v_mov_b32_e32 v0, v2 |
| ; GFX90A-NEXT: v_mov_b32_e32 v1, v4 |
| ; GFX90A-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX942-LABEL: optnone_atomicrmw_add_i64_expand: |
| ; GFX942: ; %bb.0: |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX942-NEXT: s_mov_b64 s[0:1], src_private_base |
| ; GFX942-NEXT: s_mov_b32 s2, 32 |
| ; GFX942-NEXT: s_lshr_b64 s[0:1], s[0:1], s2 |
| ; GFX942-NEXT: s_getpc_b64 s[2:3] |
| ; GFX942-NEXT: s_add_u32 s2, s2, global@rel32@lo+4 |
| ; GFX942-NEXT: s_addc_u32 s3, s3, global@rel32@hi+12 |
| ; GFX942-NEXT: s_cmp_eq_u32 s3, s0 |
| ; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0 |
| ; GFX942-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1] |
| ; GFX942-NEXT: s_mov_b64 s[0:1], -1 |
| ; GFX942-NEXT: s_mov_b32 s2, 1 |
| ; GFX942-NEXT: v_cmp_ne_u32_e64 s[2:3], v2, s2 |
| ; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3] |
| ; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GFX942-NEXT: s_cbranch_vccnz .LBB4_3 |
| ; GFX942-NEXT: .LBB4_1: ; %Flow |
| ; GFX942-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[0:1] |
| ; GFX942-NEXT: s_mov_b32 s0, 1 |
| ; GFX942-NEXT: v_cmp_ne_u32_e64 s[0:1], v4, s0 |
| ; GFX942-NEXT: s_and_b64 vcc, exec, s[0:1] |
| ; GFX942-NEXT: s_cbranch_vccnz .LBB4_4 |
| ; GFX942-NEXT: ; %bb.2: ; %atomicrmw.private |
| ; GFX942-NEXT: s_waitcnt lgkmcnt(0) |
| ; GFX942-NEXT: s_nop 1 |
| ; GFX942-NEXT: scratch_load_dwordx2 v[2:3], off, s0 |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) |
| ; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 0, v[0:1] |
| ; GFX942-NEXT: scratch_store_dwordx2 off, v[0:1], s0 |
| ; GFX942-NEXT: s_branch .LBB4_4 |
| ; GFX942-NEXT: .LBB4_3: ; %atomicrmw.global |
| ; GFX942-NEXT: s_getpc_b64 s[0:1] |
| ; GFX942-NEXT: s_add_u32 s0, s0, global@rel32@lo+4 |
| ; GFX942-NEXT: s_addc_u32 s1, s1, global@rel32@hi+12 |
| ; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[0:1] |
| ; GFX942-NEXT: flat_atomic_add_x2 v[2:3], v[2:3], v[0:1] sc0 |
| ; GFX942-NEXT: s_mov_b64 s[0:1], 0 |
| ; GFX942-NEXT: s_branch .LBB4_1 |
| ; GFX942-NEXT: .LBB4_4: ; %atomicrmw.phi |
| ; GFX942-NEXT: ; %bb.5: ; %atomicrmw.end |
| ; GFX942-NEXT: s_mov_b32 s0, 32 |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX942-NEXT: v_lshrrev_b64 v[4:5], s0, v[2:3] |
| ; GFX942-NEXT: v_mov_b32_e32 v0, v2 |
| ; GFX942-NEXT: v_mov_b32_e32 v1, v4 |
| ; GFX942-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX1100-LABEL: optnone_atomicrmw_add_i64_expand: |
| ; GFX1100: ; %bb.0: |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX1100-NEXT: s_mov_b64 s[0:1], src_private_base |
| ; GFX1100-NEXT: s_mov_b32 s2, 32 |
| ; GFX1100-NEXT: s_lshr_b64 s[0:1], s[0:1], s2 |
| ; GFX1100-NEXT: s_getpc_b64 s[2:3] |
| ; GFX1100-NEXT: s_add_u32 s2, s2, global@rel32@lo+4 |
| ; GFX1100-NEXT: s_addc_u32 s3, s3, global@rel32@hi+12 |
| ; GFX1100-NEXT: s_cmp_eq_u32 s3, s0 |
| ; GFX1100-NEXT: s_cselect_b32 s0, -1, 0 |
| ; GFX1100-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 |
| ; GFX1100-NEXT: s_mov_b32 s0, -1 |
| ; GFX1100-NEXT: s_mov_b32 s1, 1 |
| ; GFX1100-NEXT: v_cmp_ne_u32_e64 s1, v2, s1 |
| ; GFX1100-NEXT: s_and_b32 vcc_lo, exec_lo, s1 |
| ; GFX1100-NEXT: ; implicit-def: $vgpr3_vgpr4 |
| ; GFX1100-NEXT: s_cbranch_vccnz .LBB4_3 |
| ; GFX1100-NEXT: .LBB4_1: ; %Flow |
| ; GFX1100-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 |
| ; GFX1100-NEXT: s_mov_b32 s0, 1 |
| ; GFX1100-NEXT: v_cmp_ne_u32_e64 s0, v2, s0 |
| ; GFX1100-NEXT: s_and_b32 vcc_lo, exec_lo, s0 |
| ; GFX1100-NEXT: s_cbranch_vccnz .LBB4_4 |
| ; GFX1100-NEXT: ; %bb.2: ; %atomicrmw.private |
| ; GFX1100-NEXT: s_waitcnt lgkmcnt(0) |
| ; GFX1100-NEXT: scratch_load_b64 v[3:4], off, s0 |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) |
| ; GFX1100-NEXT: v_add_co_u32 v0, s0, v3, v0 |
| ; GFX1100-NEXT: v_add_co_ci_u32_e64 v1, s0, v4, v1, s0 |
| ; GFX1100-NEXT: scratch_store_b64 off, v[0:1], s0 |
| ; GFX1100-NEXT: s_branch .LBB4_4 |
| ; GFX1100-NEXT: .LBB4_3: ; %atomicrmw.global |
| ; GFX1100-NEXT: s_getpc_b64 s[0:1] |
| ; GFX1100-NEXT: s_add_u32 s0, s0, global@rel32@lo+4 |
| ; GFX1100-NEXT: s_addc_u32 s1, s1, global@rel32@hi+12 |
| ; GFX1100-NEXT: v_mov_b32_e32 v3, s1 |
| ; GFX1100-NEXT: v_mov_b32_e32 v2, s0 |
| ; GFX1100-NEXT: flat_atomic_add_u64 v[3:4], v[2:3], v[0:1] glc |
| ; GFX1100-NEXT: s_mov_b32 s0, 0 |
| ; GFX1100-NEXT: s_branch .LBB4_1 |
| ; GFX1100-NEXT: .LBB4_4: ; %atomicrmw.phi |
| ; GFX1100-NEXT: ; %bb.5: ; %atomicrmw.end |
| ; GFX1100-NEXT: s_mov_b32 s0, 32 |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX1100-NEXT: v_lshrrev_b64 v[1:2], s0, v[3:4] |
| ; GFX1100-NEXT: v_mov_b32_e32 v0, v3 |
| ; GFX1100-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX1200-LABEL: optnone_atomicrmw_add_i64_expand: |
| ; GFX1200: ; %bb.0: |
| ; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0 |
| ; GFX1200-NEXT: s_wait_expcnt 0x0 |
| ; GFX1200-NEXT: s_wait_samplecnt 0x0 |
| ; GFX1200-NEXT: s_wait_bvhcnt 0x0 |
| ; GFX1200-NEXT: s_wait_kmcnt 0x0 |
| ; GFX1200-NEXT: s_mov_b64 s[0:1], src_private_base |
| ; GFX1200-NEXT: s_mov_b32 s2, 32 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_lshr_b64 s[0:1], s[0:1], s2 |
| ; GFX1200-NEXT: s_getpc_b64 s[2:3] |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_sext_i32_i16 s3, s3 |
| ; GFX1200-NEXT: s_add_co_u32 s2, s2, global@rel32@lo+12 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_add_co_ci_u32 s3, s3, global@rel32@hi+24 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_cmp_eq_u32 s3, s0 |
| ; GFX1200-NEXT: s_cselect_b32 s0, -1, 0 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 |
| ; GFX1200-NEXT: s_mov_b32 s0, -1 |
| ; GFX1200-NEXT: s_mov_b32 s1, 1 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: v_cmp_ne_u32_e64 s1, v2, s1 |
| ; GFX1200-NEXT: s_and_b32 vcc_lo, exec_lo, s1 |
| ; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_cbranch_vccnz .LBB4_3 |
| ; GFX1200-NEXT: .LBB4_1: ; %Flow |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 |
| ; GFX1200-NEXT: s_mov_b32 s0, 1 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: v_cmp_ne_u32_e64 s0, v2, s0 |
| ; GFX1200-NEXT: s_and_b32 vcc_lo, exec_lo, s0 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_cbranch_vccnz .LBB4_4 |
| ; GFX1200-NEXT: ; %bb.2: ; %atomicrmw.private |
| ; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0 |
| ; GFX1200-NEXT: scratch_load_b64 v[3:4], off, s0 |
| ; GFX1200-NEXT: s_wait_loadcnt 0x0 |
| ; GFX1200-NEXT: v_add_co_u32 v0, s0, v3, v0 |
| ; GFX1200-NEXT: s_wait_alu 0xf1ff |
| ; GFX1200-NEXT: v_add_co_ci_u32_e64 v1, s0, v4, v1, s0 |
| ; GFX1200-NEXT: scratch_store_b64 off, v[0:1], s0 |
| ; GFX1200-NEXT: s_branch .LBB4_4 |
| ; GFX1200-NEXT: .LBB4_3: ; %atomicrmw.global |
| ; GFX1200-NEXT: s_getpc_b64 s[0:1] |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_sext_i32_i16 s1, s1 |
| ; GFX1200-NEXT: s_add_co_u32 s0, s0, global@rel32@lo+12 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_add_co_ci_u32 s1, s1, global@rel32@hi+24 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: v_mov_b32_e32 v3, s1 |
| ; GFX1200-NEXT: v_mov_b32_e32 v2, s0 |
| ; GFX1200-NEXT: flat_atomic_add_u64 v[3:4], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV |
| ; GFX1200-NEXT: s_mov_b32 s0, 0 |
| ; GFX1200-NEXT: s_branch .LBB4_1 |
| ; GFX1200-NEXT: .LBB4_4: ; %atomicrmw.phi |
| ; GFX1200-NEXT: ; %bb.5: ; %atomicrmw.end |
| ; GFX1200-NEXT: s_mov_b32 s0, 32 |
| ; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0 |
| ; GFX1200-NEXT: s_wait_alu 0xf1fe |
| ; GFX1200-NEXT: v_lshrrev_b64 v[1:2], s0, v[3:4] |
| ; GFX1200-NEXT: v_mov_b32_e32 v0, v3 |
| ; GFX1200-NEXT: s_setpc_b64 s[30:31] |
| %rmw = atomicrmw add ptr addrspacecast (ptr addrspace(1) @global to ptr), i64 %val syncscope("agent") monotonic, align 8 |
| ret i64 %rmw |
| } |
| |
| ; Make sure there is no error on an invalid addrspacecast without optimizations |
| define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 { |
| ; GFX908-LABEL: optnone_atomicrmw_fadd_f64_expand: |
| ; GFX908: ; %bb.0: |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GFX908-NEXT: s_mov_b32 s6, 32 |
| ; GFX908-NEXT: s_lshr_b64 s[4:5], s[4:5], s6 |
| ; GFX908-NEXT: s_getpc_b64 s[6:7] |
| ; GFX908-NEXT: s_add_u32 s6, s6, global@rel32@lo+4 |
| ; GFX908-NEXT: s_addc_u32 s7, s7, global@rel32@hi+12 |
| ; GFX908-NEXT: s_cmp_eq_u32 s7, s4 |
| ; GFX908-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GFX908-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5] |
| ; GFX908-NEXT: s_mov_b64 s[4:5], -1 |
| ; GFX908-NEXT: s_mov_b32 s6, 1 |
| ; GFX908-NEXT: v_readfirstlane_b32 s7, v2 |
| ; GFX908-NEXT: s_cmp_lg_u32 s7, s6 |
| ; GFX908-NEXT: s_cselect_b64 s[6:7], -1, 0 |
| ; GFX908-NEXT: s_and_b64 vcc, exec, s[6:7] |
| ; GFX908-NEXT: ; implicit-def: $vgpr3_vgpr4 |
| ; GFX908-NEXT: s_cbranch_vccnz .LBB5_2 |
| ; GFX908-NEXT: s_branch .LBB5_3 |
| ; GFX908-NEXT: .LBB5_1: ; %atomicrmw.private |
| ; GFX908-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) |
| ; GFX908-NEXT: v_mov_b32_e32 v4, v3 |
| ; GFX908-NEXT: v_add_f64 v[0:1], v[3:4], v[0:1] |
| ; GFX908-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GFX908-NEXT: buffer_store_dword v0, v0, s[0:3], 0 offen |
| ; GFX908-NEXT: s_branch .LBB5_6 |
| ; GFX908-NEXT: .LBB5_2: ; %atomicrmw.global |
| ; GFX908-NEXT: s_getpc_b64 s[4:5] |
| ; GFX908-NEXT: s_add_u32 s4, s4, global@rel32@lo+4 |
| ; GFX908-NEXT: s_addc_u32 s5, s5, global@rel32@hi+12 |
| ; GFX908-NEXT: v_mov_b32_e32 v2, s4 |
| ; GFX908-NEXT: v_mov_b32_e32 v3, s5 |
| ; GFX908-NEXT: flat_load_dwordx2 v[3:4], v[2:3] |
| ; GFX908-NEXT: s_mov_b64 s[4:5], 0 |
| ; GFX908-NEXT: s_branch .LBB5_4 |
| ; GFX908-NEXT: .LBB5_3: ; %Flow |
| ; GFX908-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GFX908-NEXT: s_cbranch_vccnz .LBB5_1 |
| ; GFX908-NEXT: s_branch .LBB5_6 |
| ; GFX908-NEXT: .LBB5_4: ; %atomicrmw.start |
| ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: v_mov_b32_e32 v6, v4 |
| ; GFX908-NEXT: v_mov_b32_e32 v5, v3 |
| ; GFX908-NEXT: v_add_f64 v[3:4], v[5:6], v[0:1] |
| ; GFX908-NEXT: s_getpc_b64 s[6:7] |
| ; GFX908-NEXT: s_add_u32 s6, s6, global@rel32@lo+4 |
| ; GFX908-NEXT: s_addc_u32 s7, s7, global@rel32@hi+12 |
| ; GFX908-NEXT: v_mov_b32_e32 v8, s7 |
| ; GFX908-NEXT: v_mov_b32_e32 v7, s6 |
| ; GFX908-NEXT: flat_atomic_cmpswap_x2 v[3:4], v[7:8], v[3:6] glc |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX908-NEXT: v_cmp_eq_u64_e64 s[6:7], v[3:4], v[5:6] |
| ; GFX908-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5] |
| ; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GFX908-NEXT: s_cbranch_execnz .LBB5_4 |
| ; GFX908-NEXT: ; %bb.5: ; %atomicrmw.end1 |
| ; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GFX908-NEXT: s_mov_b64 s[4:5], 0 |
| ; GFX908-NEXT: s_branch .LBB5_3 |
| ; GFX908-NEXT: .LBB5_6: ; %atomicrmw.phi |
| ; GFX908-NEXT: ; %bb.7: ; %atomicrmw.end |
| ; GFX908-NEXT: s_mov_b32 s4, 32 |
| ; GFX908-NEXT: v_lshrrev_b64 v[1:2], s4, v[3:4] |
| ; GFX908-NEXT: v_mov_b32_e32 v0, v3 |
| ; GFX908-NEXT: s_waitcnt vmcnt(0) |
| ; GFX908-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX90A-LABEL: optnone_atomicrmw_fadd_f64_expand: |
| ; GFX90A: ; %bb.0: |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base |
| ; GFX90A-NEXT: s_mov_b32 s6, 32 |
| ; GFX90A-NEXT: s_lshr_b64 s[4:5], s[4:5], s6 |
| ; GFX90A-NEXT: s_getpc_b64 s[6:7] |
| ; GFX90A-NEXT: s_add_u32 s6, s6, global@rel32@lo+4 |
| ; GFX90A-NEXT: s_addc_u32 s7, s7, global@rel32@hi+12 |
| ; GFX90A-NEXT: s_cmp_eq_u32 s7, s4 |
| ; GFX90A-NEXT: s_cselect_b64 s[4:5], -1, 0 |
| ; GFX90A-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5] |
| ; GFX90A-NEXT: s_mov_b64 s[4:5], -1 |
| ; GFX90A-NEXT: s_mov_b32 s6, 1 |
| ; GFX90A-NEXT: v_readfirstlane_b32 s7, v2 |
| ; GFX90A-NEXT: s_cmp_lg_u32 s7, s6 |
| ; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0 |
| ; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7] |
| ; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GFX90A-NEXT: s_cbranch_vccnz .LBB5_2 |
| ; GFX90A-NEXT: s_branch .LBB5_3 |
| ; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.private |
| ; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) |
| ; GFX90A-NEXT: v_mov_b32_e32 v3, v2 |
| ; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[0:1] |
| ; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen |
| ; GFX90A-NEXT: buffer_store_dword v0, v0, s[0:3], 0 offen |
| ; GFX90A-NEXT: s_branch .LBB5_6 |
| ; GFX90A-NEXT: .LBB5_2: ; %atomicrmw.global |
| ; GFX90A-NEXT: s_getpc_b64 s[4:5] |
| ; GFX90A-NEXT: s_add_u32 s4, s4, global@rel32@lo+4 |
| ; GFX90A-NEXT: s_addc_u32 s5, s5, global@rel32@hi+12 |
| ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] |
| ; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[2:3] |
| ; GFX90A-NEXT: s_mov_b64 s[4:5], 0 |
| ; GFX90A-NEXT: s_branch .LBB5_4 |
| ; GFX90A-NEXT: .LBB5_3: ; %Flow |
| ; GFX90A-NEXT: s_and_b64 vcc, exec, s[4:5] |
| ; GFX90A-NEXT: s_cbranch_vccnz .LBB5_1 |
| ; GFX90A-NEXT: s_branch .LBB5_6 |
| ; GFX90A-NEXT: .LBB5_4: ; %atomicrmw.start |
| ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1] |
| ; GFX90A-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] |
| ; GFX90A-NEXT: s_getpc_b64 s[6:7] |
| ; GFX90A-NEXT: s_add_u32 s6, s6, global@rel32@lo+4 |
| ; GFX90A-NEXT: s_addc_u32 s7, s7, global@rel32@hi+12 |
| ; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1] |
| ; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[6:7], v[2:5] glc |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX90A-NEXT: v_cmp_eq_u64_e64 s[6:7], v[2:3], v[4:5] |
| ; GFX90A-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5] |
| ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5] |
| ; GFX90A-NEXT: s_cbranch_execnz .LBB5_4 |
| ; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.end1 |
| ; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] |
| ; GFX90A-NEXT: s_mov_b64 s[4:5], 0 |
| ; GFX90A-NEXT: s_branch .LBB5_3 |
| ; GFX90A-NEXT: .LBB5_6: ; %atomicrmw.phi |
| ; GFX90A-NEXT: ; %bb.7: ; %atomicrmw.end |
| ; GFX90A-NEXT: s_mov_b32 s4, 32 |
| ; GFX90A-NEXT: v_lshrrev_b64 v[4:5], s4, v[2:3] |
| ; GFX90A-NEXT: v_mov_b32_e32 v0, v2 |
| ; GFX90A-NEXT: v_mov_b32_e32 v1, v4 |
| ; GFX90A-NEXT: s_waitcnt vmcnt(0) |
| ; GFX90A-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX942-LABEL: optnone_atomicrmw_fadd_f64_expand: |
| ; GFX942: ; %bb.0: |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX942-NEXT: s_mov_b64 s[0:1], src_private_base |
| ; GFX942-NEXT: s_mov_b32 s2, 32 |
| ; GFX942-NEXT: s_lshr_b64 s[0:1], s[0:1], s2 |
| ; GFX942-NEXT: s_getpc_b64 s[2:3] |
| ; GFX942-NEXT: s_add_u32 s2, s2, global@rel32@lo+4 |
| ; GFX942-NEXT: s_addc_u32 s3, s3, global@rel32@hi+12 |
| ; GFX942-NEXT: s_cmp_eq_u32 s3, s0 |
| ; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0 |
| ; GFX942-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1] |
| ; GFX942-NEXT: s_mov_b64 s[0:1], -1 |
| ; GFX942-NEXT: s_mov_b32 s2, 1 |
| ; GFX942-NEXT: v_readfirstlane_b32 s3, v2 |
| ; GFX942-NEXT: s_cmp_lg_u32 s3, s2 |
| ; GFX942-NEXT: s_cselect_b64 s[2:3], -1, 0 |
| ; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3] |
| ; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3 |
| ; GFX942-NEXT: s_cbranch_vccnz .LBB5_2 |
| ; GFX942-NEXT: s_branch .LBB5_3 |
| ; GFX942-NEXT: .LBB5_1: ; %atomicrmw.private |
| ; GFX942-NEXT: scratch_load_dwordx2 v[2:3], off, s0 |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) |
| ; GFX942-NEXT: v_add_f64 v[0:1], v[2:3], v[0:1] |
| ; GFX942-NEXT: scratch_store_dwordx2 off, v[0:1], s0 |
| ; GFX942-NEXT: s_branch .LBB5_6 |
| ; GFX942-NEXT: .LBB5_2: ; %atomicrmw.global |
| ; GFX942-NEXT: s_getpc_b64 s[0:1] |
| ; GFX942-NEXT: s_add_u32 s0, s0, global@rel32@lo+4 |
| ; GFX942-NEXT: s_addc_u32 s1, s1, global@rel32@hi+12 |
| ; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[0:1] |
| ; GFX942-NEXT: flat_load_dwordx2 v[2:3], v[2:3] |
| ; GFX942-NEXT: s_mov_b64 s[0:1], 0 |
| ; GFX942-NEXT: s_branch .LBB5_4 |
| ; GFX942-NEXT: .LBB5_3: ; %Flow |
| ; GFX942-NEXT: s_and_b64 vcc, exec, s[0:1] |
| ; GFX942-NEXT: s_cbranch_vccnz .LBB5_1 |
| ; GFX942-NEXT: s_branch .LBB5_6 |
| ; GFX942-NEXT: .LBB5_4: ; %atomicrmw.start |
| ; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3] |
| ; GFX942-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] |
| ; GFX942-NEXT: s_getpc_b64 s[2:3] |
| ; GFX942-NEXT: s_add_u32 s2, s2, global@rel32@lo+4 |
| ; GFX942-NEXT: s_addc_u32 s3, s3, global@rel32@hi+12 |
| ; GFX942-NEXT: v_mov_b64_e32 v[6:7], s[2:3] |
| ; GFX942-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[6:7], v[2:5] sc0 sc1 |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX942-NEXT: v_cmp_eq_u64_e64 s[2:3], v[2:3], v[4:5] |
| ; GFX942-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1] |
| ; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1] |
| ; GFX942-NEXT: s_cbranch_execnz .LBB5_4 |
| ; GFX942-NEXT: ; %bb.5: ; %atomicrmw.end1 |
| ; GFX942-NEXT: s_or_b64 exec, exec, s[0:1] |
| ; GFX942-NEXT: s_mov_b64 s[0:1], 0 |
| ; GFX942-NEXT: s_branch .LBB5_3 |
| ; GFX942-NEXT: .LBB5_6: ; %atomicrmw.phi |
| ; GFX942-NEXT: ; %bb.7: ; %atomicrmw.end |
| ; GFX942-NEXT: s_mov_b32 s0, 32 |
| ; GFX942-NEXT: v_lshrrev_b64 v[4:5], s0, v[2:3] |
| ; GFX942-NEXT: v_mov_b32_e32 v0, v2 |
| ; GFX942-NEXT: v_mov_b32_e32 v1, v4 |
| ; GFX942-NEXT: s_waitcnt vmcnt(0) |
| ; GFX942-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX1100-LABEL: optnone_atomicrmw_fadd_f64_expand: |
| ; GFX1100: ; %bb.0: |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX1100-NEXT: s_mov_b64 s[0:1], src_private_base |
| ; GFX1100-NEXT: s_mov_b32 s2, 32 |
| ; GFX1100-NEXT: s_lshr_b64 s[0:1], s[0:1], s2 |
| ; GFX1100-NEXT: s_getpc_b64 s[2:3] |
| ; GFX1100-NEXT: s_add_u32 s2, s2, global@rel32@lo+4 |
| ; GFX1100-NEXT: s_addc_u32 s3, s3, global@rel32@hi+12 |
| ; GFX1100-NEXT: s_cmp_eq_u32 s3, s0 |
| ; GFX1100-NEXT: s_cselect_b32 s0, -1, 0 |
| ; GFX1100-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 |
| ; GFX1100-NEXT: s_mov_b32 s0, -1 |
| ; GFX1100-NEXT: s_mov_b32 s1, 1 |
| ; GFX1100-NEXT: v_cmp_ne_u32_e64 s1, v2, s1 |
| ; GFX1100-NEXT: s_and_b32 vcc_lo, exec_lo, s1 |
| ; GFX1100-NEXT: ; implicit-def: $vgpr3_vgpr4 |
| ; GFX1100-NEXT: s_cbranch_vccnz .LBB5_2 |
| ; GFX1100-NEXT: s_branch .LBB5_3 |
| ; GFX1100-NEXT: .LBB5_1: ; %atomicrmw.private |
| ; GFX1100-NEXT: scratch_load_b64 v[3:4], off, s0 |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) |
| ; GFX1100-NEXT: v_add_f64 v[0:1], v[3:4], v[0:1] |
| ; GFX1100-NEXT: scratch_store_b64 off, v[0:1], s0 |
| ; GFX1100-NEXT: s_branch .LBB5_6 |
| ; GFX1100-NEXT: .LBB5_2: ; %atomicrmw.global |
| ; GFX1100-NEXT: s_getpc_b64 s[0:1] |
| ; GFX1100-NEXT: s_add_u32 s0, s0, global@rel32@lo+4 |
| ; GFX1100-NEXT: s_addc_u32 s1, s1, global@rel32@hi+12 |
| ; GFX1100-NEXT: v_mov_b32_e32 v3, s1 |
| ; GFX1100-NEXT: v_mov_b32_e32 v2, s0 |
| ; GFX1100-NEXT: flat_load_b64 v[3:4], v[2:3] |
| ; GFX1100-NEXT: s_mov_b32 s0, 0 |
| ; GFX1100-NEXT: s_branch .LBB5_4 |
| ; GFX1100-NEXT: .LBB5_3: ; %Flow |
| ; GFX1100-NEXT: s_and_b32 vcc_lo, exec_lo, s0 |
| ; GFX1100-NEXT: s_cbranch_vccnz .LBB5_1 |
| ; GFX1100-NEXT: s_branch .LBB5_6 |
| ; GFX1100-NEXT: .LBB5_4: ; %atomicrmw.start |
| ; GFX1100-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX1100-NEXT: v_mov_b32_e32 v6, v4 |
| ; GFX1100-NEXT: v_mov_b32_e32 v5, v3 |
| ; GFX1100-NEXT: v_add_f64 v[3:4], v[5:6], v[0:1] |
| ; GFX1100-NEXT: s_getpc_b64 s[2:3] |
| ; GFX1100-NEXT: s_add_u32 s2, s2, global@rel32@lo+4 |
| ; GFX1100-NEXT: s_addc_u32 s3, s3, global@rel32@hi+12 |
| ; GFX1100-NEXT: v_mov_b32_e32 v8, s3 |
| ; GFX1100-NEXT: v_mov_b32_e32 v7, s2 |
| ; GFX1100-NEXT: flat_atomic_cmpswap_b64 v[3:4], v[7:8], v[3:6] glc |
| ; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) |
| ; GFX1100-NEXT: v_cmp_eq_u64_e64 s1, v[3:4], v[5:6] |
| ; GFX1100-NEXT: s_or_b32 s0, s1, s0 |
| ; GFX1100-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 |
| ; GFX1100-NEXT: s_cbranch_execnz .LBB5_4 |
| ; GFX1100-NEXT: ; %bb.5: ; %atomicrmw.end1 |
| ; GFX1100-NEXT: s_or_b32 exec_lo, exec_lo, s0 |
| ; GFX1100-NEXT: s_mov_b32 s0, 0 |
| ; GFX1100-NEXT: s_branch .LBB5_3 |
| ; GFX1100-NEXT: .LBB5_6: ; %atomicrmw.phi |
| ; GFX1100-NEXT: ; %bb.7: ; %atomicrmw.end |
| ; GFX1100-NEXT: s_mov_b32 s0, 32 |
| ; GFX1100-NEXT: v_lshrrev_b64 v[1:2], s0, v[3:4] |
| ; GFX1100-NEXT: v_mov_b32_e32 v0, v3 |
| ; GFX1100-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX1200-LABEL: optnone_atomicrmw_fadd_f64_expand: |
| ; GFX1200: ; %bb.0: |
| ; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0 |
| ; GFX1200-NEXT: s_wait_expcnt 0x0 |
| ; GFX1200-NEXT: s_wait_samplecnt 0x0 |
| ; GFX1200-NEXT: s_wait_bvhcnt 0x0 |
| ; GFX1200-NEXT: s_wait_kmcnt 0x0 |
| ; GFX1200-NEXT: s_mov_b64 s[0:1], src_private_base |
| ; GFX1200-NEXT: s_mov_b32 s2, 32 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_lshr_b64 s[0:1], s[0:1], s2 |
| ; GFX1200-NEXT: s_getpc_b64 s[2:3] |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_sext_i32_i16 s3, s3 |
| ; GFX1200-NEXT: s_add_co_u32 s2, s2, global@rel32@lo+12 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_add_co_ci_u32 s3, s3, global@rel32@hi+24 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_cmp_eq_u32 s3, s0 |
| ; GFX1200-NEXT: s_cselect_b32 s0, -1, 0 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 |
| ; GFX1200-NEXT: s_mov_b32 s0, -1 |
| ; GFX1200-NEXT: s_mov_b32 s1, 1 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: v_cmp_ne_u32_e64 s1, v2, s1 |
| ; GFX1200-NEXT: s_and_b32 vcc_lo, exec_lo, s1 |
| ; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_cbranch_vccnz .LBB5_2 |
| ; GFX1200-NEXT: s_branch .LBB5_3 |
| ; GFX1200-NEXT: .LBB5_1: ; %atomicrmw.private |
| ; GFX1200-NEXT: scratch_load_b64 v[3:4], off, s0 |
| ; GFX1200-NEXT: s_wait_loadcnt 0x0 |
| ; GFX1200-NEXT: v_add_f64_e64 v[0:1], v[3:4], v[0:1] |
| ; GFX1200-NEXT: scratch_store_b64 off, v[0:1], s0 |
| ; GFX1200-NEXT: s_branch .LBB5_6 |
| ; GFX1200-NEXT: .LBB5_2: ; %atomicrmw.global |
| ; GFX1200-NEXT: s_getpc_b64 s[0:1] |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_sext_i32_i16 s1, s1 |
| ; GFX1200-NEXT: s_add_co_u32 s0, s0, global@rel32@lo+12 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_add_co_ci_u32 s1, s1, global@rel32@hi+24 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: v_mov_b32_e32 v3, s1 |
| ; GFX1200-NEXT: v_mov_b32_e32 v2, s0 |
| ; GFX1200-NEXT: flat_load_b64 v[3:4], v[2:3] |
| ; GFX1200-NEXT: s_mov_b32 s0, 0 |
| ; GFX1200-NEXT: s_branch .LBB5_4 |
| ; GFX1200-NEXT: .LBB5_3: ; %Flow |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_and_b32 vcc_lo, exec_lo, s0 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_cbranch_vccnz .LBB5_1 |
| ; GFX1200-NEXT: s_branch .LBB5_6 |
| ; GFX1200-NEXT: .LBB5_4: ; %atomicrmw.start |
| ; GFX1200-NEXT: ; =>This Inner Loop Header: Depth=1 |
| ; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0 |
| ; GFX1200-NEXT: v_mov_b32_e32 v6, v4 |
| ; GFX1200-NEXT: v_mov_b32_e32 v5, v3 |
| ; GFX1200-NEXT: v_add_f64_e64 v[3:4], v[5:6], v[0:1] |
| ; GFX1200-NEXT: s_getpc_b64 s[2:3] |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_sext_i32_i16 s3, s3 |
| ; GFX1200-NEXT: s_add_co_u32 s2, s2, global@rel32@lo+12 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_add_co_ci_u32 s3, s3, global@rel32@hi+24 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: v_mov_b32_e32 v8, s3 |
| ; GFX1200-NEXT: v_mov_b32_e32 v7, s2 |
| ; GFX1200-NEXT: flat_atomic_cmpswap_b64 v[3:4], v[7:8], v[3:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS |
| ; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0 |
| ; GFX1200-NEXT: v_cmp_eq_u64_e64 s1, v[3:4], v[5:6] |
| ; GFX1200-NEXT: s_or_b32 s0, s1, s0 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 |
| ; GFX1200-NEXT: s_cbranch_execnz .LBB5_4 |
| ; GFX1200-NEXT: ; %bb.5: ; %atomicrmw.end1 |
| ; GFX1200-NEXT: s_or_b32 exec_lo, exec_lo, s0 |
| ; GFX1200-NEXT: s_mov_b32 s0, 0 |
| ; GFX1200-NEXT: s_branch .LBB5_3 |
| ; GFX1200-NEXT: .LBB5_6: ; %atomicrmw.phi |
| ; GFX1200-NEXT: ; %bb.7: ; %atomicrmw.end |
| ; GFX1200-NEXT: s_mov_b32 s0, 32 |
| ; GFX1200-NEXT: s_wait_alu 0xfffe |
| ; GFX1200-NEXT: v_lshrrev_b64 v[1:2], s0, v[3:4] |
| ; GFX1200-NEXT: v_mov_b32_e32 v0, v3 |
| ; GFX1200-NEXT: s_setpc_b64 s[30:31] |
| %rmw = atomicrmw fadd ptr addrspacecast (ptr addrspace(1) @global to ptr), double %val monotonic, align 8 |
| ret double %rmw |
| } |
| |
| attributes #0 = { nounwind } |
| attributes #1 = { noinline nounwind optnone } |
| |
| !0 = !{} |
