blob: 074b7074e1952572bfaef6c42225b96bfb1ff7dd [file] [edit]
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=amdgpu12.50-amd-amdhsa -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
# The only asyncmark that reaches each wait.asyncmark below comes around the
# loop backedge, so the first visit of the loop body sees an empty sequence.
# The wait must still be generated once the backedge state is merged in, like
# the loop-carried soft waitcnts of https://github.com/llvm/llvm-project/pull/220357.
# The wait.asyncmark is never erased, so every visit recomputes its wait from
# the merged state.
---
name: loop_carried_mark
tracksRegLiveness: true
body: |
; CHECK-LABEL: name: loop_carried_mark
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; CHECK-NEXT: S_WAIT_KMCNT 0
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS
; CHECK-NEXT: S_WAIT_DSCNT 0
; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
; CHECK-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
; CHECK-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: S_ENDPGM 0
bb.0:
successors: %bb.1
liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
S_BRANCH %bb.1
bb.1:
successors: %bb.1, %bb.2
liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
WAIT_ASYNCMARK 0, 2
$vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
ASYNCMARK 2
S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
S_CBRANCH_SCC1 %bb.1, implicit $scc
S_BRANCH %bb.2
bb.2:
S_ENDPGM 0
...
---
name: loop_carried_mark_with_soft_waits
tracksRegLiveness: true
body: |
; CHECK-LABEL: name: loop_carried_mark_with_soft_waits
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; CHECK-NEXT: S_WAIT_KMCNT 0
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_WAIT_DSCNT 0
; CHECK-NEXT: S_BARRIER
; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS
; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
; CHECK-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
; CHECK-NEXT: S_WAIT_DSCNT 0
; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr2, $vgpr3, 4, 0, implicit $exec :: (store (s32), addrspace 3)
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
; CHECK-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: S_ENDPGM 0
bb.0:
successors: %bb.1
liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
S_BRANCH %bb.1
bb.1:
successors: %bb.1, %bb.2
liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
S_WAIT_DSCNT_soft 0
S_BARRIER
S_WAIT_DSCNT_soft 0
WAIT_ASYNCMARK 0, 2
$vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
DS_WRITE_B32_gfx9 $vgpr2, $vgpr3, 4, 0, implicit $exec :: (store (s32), addrspace 3)
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
ASYNCMARK 2
S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
S_CBRANCH_SCC1 %bb.1, implicit $scc
S_BRANCH %bb.2
bb.2:
S_ENDPGM 0
...
---
name: loop_carried_mark_other_stage
tracksRegLiveness: true
body: |
; CHECK-LABEL: name: loop_carried_mark_other_stage
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; CHECK-NEXT: S_WAIT_KMCNT 0
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS
; CHECK-NEXT: S_WAIT_DSCNT 0
; CHECK-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
; CHECK-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B32 $vgpr0_vgpr1, $vgpr2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 3), (store (s32), addrspace 1)
; CHECK-NEXT: ASYNCMARK .GLOBAL_STORE_ASYNC_FROM_LDS
; CHECK-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: S_ENDPGM 0
bb.0:
successors: %bb.1
liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
S_BRANCH %bb.1
bb.1:
successors: %bb.1, %bb.2
liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
WAIT_ASYNCMARK 0, 2
$vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
GLOBAL_STORE_ASYNC_FROM_LDS_B32 $vgpr0_vgpr1, $vgpr2, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 3), (store (s32), addrspace 1)
ASYNCMARK 8
S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
S_CBRANCH_SCC1 %bb.1, implicit $scc
S_BRANCH %bb.2
bb.2:
S_ENDPGM 0
...
# A soft wait directly before the wait.asyncmark looks redundant on the first
# visit and is queued for erasure, while the wait.asyncmark finds no marks yet.
# On the revisit both are needed: dscnt for the loop-carried DS write, and
# asynccnt for the loop-carried mark.
---
name: loop_carried_mark_adjacent_soft_wait
tracksRegLiveness: true
body: |
; CHECK-LABEL: name: loop_carried_mark_adjacent_soft_wait
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; CHECK-NEXT: S_WAIT_KMCNT 0
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_WAIT_DSCNT 0
; CHECK-NEXT: WAIT_ASYNCMARK 0, .GLOBAL_LOAD_ASYNC_TO_LDS
; CHECK-NEXT: S_WAIT_ASYNCCNT 0, implicit-def $asynccnt, implicit $asynccnt
; CHECK-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
; CHECK-NEXT: S_WAIT_DSCNT 0
; CHECK-NEXT: DS_WRITE_B32_gfx9 $vgpr2, $vgpr3, 4, 0, implicit $exec :: (store (s32), addrspace 3)
; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
; CHECK-NEXT: ASYNCMARK .GLOBAL_LOAD_ASYNC_TO_LDS
; CHECK-NEXT: S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: S_ENDPGM 0
bb.0:
successors: %bb.1
liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
S_BRANCH %bb.1
bb.1:
successors: %bb.1, %bb.2
liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0
S_WAIT_DSCNT_soft 0
WAIT_ASYNCMARK 0, 2
$vgpr3 = DS_READ_B32_gfx9 $vgpr2, 0, 0, implicit $exec :: (load (s32), addrspace 3)
DS_WRITE_B32_gfx9 $vgpr2, $vgpr3, 4, 0, implicit $exec :: (store (s32), addrspace 3)
GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load (s32), addrspace 1), (store (s32), addrspace 3)
ASYNCMARK 2
S_CMP_EQ_U32 $sgpr0, 0, implicit-def $scc
S_CBRANCH_SCC1 %bb.1, implicit $scc
S_BRANCH %bb.2
bb.2:
S_ENDPGM 0
...