| # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 | |
| # RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-waitcnts -o - %s | FileCheck %s | |
| # The only asyncmark that reaches each wait.asyncmark below comes around the | |
| # loop backedge, so the first visit of the loop body sees an empty sequence. | |
| # The wait must still be generated once the backedge state is merged in, like | |
| # the loop-carried soft waitcnts of https://github.com/llvm/llvm-project/pull/220357. | |
| # The wait.asyncmark is never erased, so every visit recomputes its wait from | |
| # the merged state. | |
| --- | |
| name: loop_carried_mark | |
| tracksRegLiveness: true | |
| body: | | |
| ; CHECK-LABEL: name: loop_carried_mark | |
| ; CHECK: bb.0: | |
| ; CHECK-NEXT: successors: %bb.1(0x80000000) | |
| ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0 | |
| ; CHECK-NEXT: S_WAIT_KMCNT 0 | |
| ; CHECK-NEXT: S_BRANCH %bb.1 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: bb.1: | |
| ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) | |
| ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS | |
| ; CHECK-NEXT: S_WAIT_DSCNT 0 | |
| ; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt | |
| ; CHECK-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3) | |
| ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3) | |
| ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS | |
| ; CHECK-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc | |
| ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc | |
| ; CHECK-NEXT: S_BRANCH %bb.2 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: bb.2: | |
| ; CHECK-NEXT: S_ENDPGM 0 | |
| bb.0: | |
| successors: %bb.1 | |
| liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| S_BRANCH %bb.1 | |
| bb.1: | |
| successors: %bb.1, %bb.2 | |
| liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| WAIT_ASYNCMARK 0, 2 | |
| $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3) | |
| GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3) | |
| ASYNCMARK 2 | |
| S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc | |
| S_CBRANCH_SCC1 %bb.1, implicit $scc | |
| S_BRANCH %bb.2 | |
| bb.2: | |
| S_ENDPGM 0 | |
| ... | |
| --- | |
| name: loop_carried_mark_with_soft_waits | |
| tracksRegLiveness: true | |
| body: | | |
| ; CHECK-LABEL: name: loop_carried_mark_with_soft_waits | |
| ; CHECK: bb.0: | |
| ; CHECK-NEXT: successors: %bb.1(0x80000000) | |
| ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0 | |
| ; CHECK-NEXT: S_WAIT_KMCNT 0 | |
| ; CHECK-NEXT: S_BRANCH %bb.1 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: bb.1: | |
| ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) | |
| ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: S_WAIT_DSCNT 0 | |
| ; CHECK-NEXT: S_BARRIER | |
| ; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS | |
| ; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt | |
| ; CHECK-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3) | |
| ; CHECK-NEXT: S_WAIT_DSCNT 0 | |
| ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr2, $vgpr3, 4, 0, implicit $exec :: (store (s32), addrspace 3) | |
| ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3) | |
| ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS | |
| ; CHECK-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc | |
| ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc | |
| ; CHECK-NEXT: S_BRANCH %bb.2 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: bb.2: | |
| ; CHECK-NEXT: S_ENDPGM 0 | |
| bb.0: | |
| successors: %bb.1 | |
| liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| S_BRANCH %bb.1 | |
| bb.1: | |
| successors: %bb.1, %bb.2 | |
| liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| S_WAIT_DSCNT_soft 0 | |
| S_BARRIER | |
| S_WAIT_DSCNT_soft 0 | |
| WAIT_ASYNCMARK 0, 2 | |
| $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3) | |
| DS_WRITE_B32_gfx9 $vgpr2, $vgpr3, 4, 0, implicit $exec :: (store (s32), addrspace 3) | |
| GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3) | |
| ASYNCMARK 2 | |
| S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc | |
| S_CBRANCH_SCC1 %bb.1, implicit $scc | |
| S_BRANCH %bb.2 | |
| bb.2: | |
| S_ENDPGM 0 | |
| ... | |
| --- | |
| name: loop_carried_mark_other_stage | |
| tracksRegLiveness: true | |
| body: | | |
| ; CHECK-LABEL: name: loop_carried_mark_other_stage | |
| ; CHECK: bb.0: | |
| ; CHECK-NEXT: successors: %bb.1(0x80000000) | |
| ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0 | |
| ; CHECK-NEXT: S_WAIT_KMCNT 0 | |
| ; CHECK-NEXT: S_BRANCH %bb.1 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: bb.1: | |
| ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) | |
| ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS | |
| ; CHECK-NEXT: S_WAIT_DSCNT 0 | |
| ; CHECK-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3) | |
| ; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B32 $vgpr0_vgpr1, $vgpr2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 3), (store (s32), addrspace 1) | |
| ; CHECK-NEXT: ASYNCMARK .GLOBAL_STORE_ASYNC_FROM_LDS | |
| ; CHECK-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc | |
| ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc | |
| ; CHECK-NEXT: S_BRANCH %bb.2 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: bb.2: | |
| ; CHECK-NEXT: S_ENDPGM 0 | |
| bb.0: | |
| successors: %bb.1 | |
| liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| S_BRANCH %bb.1 | |
| bb.1: | |
| successors: %bb.1, %bb.2 | |
| liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| WAIT_ASYNCMARK 0, 2 | |
| $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3) | |
| GLOBAL_STORE_ASYNC_FROM_LDS_B32 $vgpr0_vgpr1, $vgpr2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 3), (store (s32), addrspace 1) | |
| ASYNCMARK 8 | |
| S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc | |
| S_CBRANCH_SCC1 %bb.1, implicit $scc | |
| S_BRANCH %bb.2 | |
| bb.2: | |
| S_ENDPGM 0 | |
| ... | |
| # A soft wait directly before the wait.asyncmark looks redundant on the first | |
| # visit and is queued for erasure, while the wait.asyncmark finds no marks yet. | |
| # On the revisit both are needed: dscnt for the loop-carried DS write, and | |
| # asynccnt for the loop-carried mark. | |
| --- | |
| name: loop_carried_mark_adjacent_soft_wait | |
| tracksRegLiveness: true | |
| body: | | |
| ; CHECK-LABEL: name: loop_carried_mark_adjacent_soft_wait | |
| ; CHECK: bb.0: | |
| ; CHECK-NEXT: successors: %bb.1(0x80000000) | |
| ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0 | |
| ; CHECK-NEXT: S_WAIT_KMCNT 0 | |
| ; CHECK-NEXT: S_BRANCH %bb.1 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: bb.1: | |
| ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) | |
| ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: S_WAIT_DSCNT 0 | |
| ; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS | |
| ; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt | |
| ; CHECK-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3) | |
| ; CHECK-NEXT: S_WAIT_DSCNT 0 | |
| ; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr2, $vgpr3, 4, 0, implicit $exec :: (store (s32), addrspace 3) | |
| ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3) | |
| ; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS | |
| ; CHECK-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc | |
| ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc | |
| ; CHECK-NEXT: S_BRANCH %bb.2 | |
| ; CHECK-NEXT: {{ $}} | |
| ; CHECK-NEXT: bb.2: | |
| ; CHECK-NEXT: S_ENDPGM 0 | |
| bb.0: | |
| successors: %bb.1 | |
| liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| S_BRANCH %bb.1 | |
| bb.1: | |
| successors: %bb.1, %bb.2 | |
| liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0 | |
| S_WAIT_DSCNT_soft 0 | |
| WAIT_ASYNCMARK 0, 2 | |
| $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3) | |
| DS_WRITE_B32_gfx9 $vgpr2, $vgpr3, 4, 0, implicit $exec :: (store (s32), addrspace 3) | |
| GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3) | |
| ASYNCMARK 2 | |
| S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc | |
| S_CBRANCH_SCC1 %bb.1, implicit $scc | |
| S_BRANCH %bb.2 | |
| bb.2: | |
| S_ENDPGM 0 | |
| ... |