| # RUN: llc -march=amdgcn -mcpu=gfx803 -run-pass si-insert-waitcnts %s -o - | FileCheck -check-prefixes=CHECK,GFX89 %s |
| # RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass si-insert-waitcnts %s -o - | FileCheck -check-prefixes=CHECK,GFX89 %s |
| |
| --- | |
| define amdgpu_kernel void @flat_zero_waitcnt(i32 addrspace(1)* %global4, |
| <4 x i32> addrspace(1)* %global16, |
| i32* %flat4, |
| <4 x i32>* %flat16) { |
| ret void |
| } |
| |
| define amdgpu_kernel void @single_fallthrough_successor_no_end_block_wait() { |
| ret void |
| } |
| |
| define amdgpu_kernel void @single_branch_successor_not_next_block() { |
| ret void |
| } |
| |
| define amdgpu_kernel void @preexisting_waitcnt() { |
| ret void |
| } |
| |
| define amdgpu_kernel void @bundle_no_waitcnt() { |
| ret void |
| } |
| |
| define amdgpu_kernel void @preexisting_waitcnt_in_bundle() { |
| ret void |
| } |
| |
| define amdgpu_kernel void @insert_in_bundle() { |
| ret void |
| } |
| |
| define amdgpu_kernel void @exit_bundle() { |
| ret void |
| } |
| |
| define amdgpu_kernel void @cross_bundle() { |
| ret void |
| } |
| |
| define amdgpu_kernel void @subregs16bit() { |
| ret void |
| } |
| ... |
| --- |
| |
| # CHECK-LABEL: name: flat_zero_waitcnt |
| |
| # CHECK-LABEL: bb.0: |
| # CHECK: FLAT_LOAD_DWORD |
| # CHECK: FLAT_LOAD_DWORDX4 |
| # Global loads will return in order so we should: |
| # s_waitcnt vmcnt(1) |
| # CHECK-NEXT: S_WAITCNT 3953 |
| |
| # CHECK-LABEL: bb.1: |
| # CHECK: FLAT_LOAD_DWORD |
| # s_waitcnt vmcnt(0) |
| # GFX89: S_WAITCNT 3952 |
| # CHECK: FLAT_LOAD_DWORDX4 |
| |
| # CHECK-LABEL: bb.2: |
| # CHECK: FLAT_LOAD_DWORD |
| # s_waitcnt vmcnt(0) |
| # GFX89: S_WAITCNT 3952 |
| # CHECK: FLAT_LOAD_DWORDX4 |
| |
| # CHECK-LABEL: bb.3: |
| # s_waitcnt vmcnt(0) |
| # GFX89: S_WAITCNT 3952 |
| # CHECK: FLAT_LOAD_DWORD |
| # CHECK: FLAT_LOAD_DWORD |
| # s_waitcnt vmcnt(0) lgkmcnt(0) |
| # GFX89: S_WAITCNT 112 |
| |
| # CHECK-LABEL: bb.4: |
| # GFX89-NOT: S_WAITCNT |
| # CHECK: FLAT_LOAD_DWORD |
| # s_waitcnt vmcnt(0) lgkmcnt(0) |
| # GFX89: S_WAITCNT 112 |
| |
| name: flat_zero_waitcnt |
| |
| body: | |
| bb.0: |
| successors: %bb.1 |
| $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %ir.global4) |
| $vgpr3_vgpr4_vgpr5_vgpr6 = FLAT_LOAD_DWORDX4 $vgpr7_vgpr8, 0, 0, implicit $exec, implicit $flat_scr :: (load (s128) from %ir.global16) |
| $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec |
| S_BRANCH %bb.1 |
| |
| bb.1: |
| successors: %bb.2 |
| $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr |
| $vgpr3_vgpr4_vgpr5_vgpr6 = FLAT_LOAD_DWORDX4 $vgpr7_vgpr8, 0, 0, implicit $exec, implicit $flat_scr :: (load (s128) from %ir.global16) |
| $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec |
| S_BRANCH %bb.2 |
| |
| bb.2: |
| successors: %bb.3 |
| $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %ir.flat4) |
| $vgpr3_vgpr4_vgpr5_vgpr6 = FLAT_LOAD_DWORDX4 $vgpr7_vgpr8, 0, 0, implicit $exec, implicit $flat_scr :: (load (s128) from %ir.flat16) |
| $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec |
| S_BRANCH %bb.3 |
| |
| bb.3: |
| successors: %bb.4 |
| $vgpr3 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %ir.flat4) |
| $vgpr4 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %ir.global4) |
| $vgpr0 = V_MOV_B32_e32 $vgpr3, implicit $exec |
| S_BRANCH %bb.4 |
| |
| bb.4: |
| $vgpr5 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %ir.flat4) |
| $vgpr0 = V_MOV_B32_e32 $vgpr5, implicit $exec |
| S_ENDPGM 0 |
| ... |
| --- |
| # There is only a single fallthrough successor block, so there's no |
| # need to wait immediately. |
| |
| # CHECK-LABEL: name: single_fallthrough_successor_no_end_block_wait |
| # CHECK: $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2 |
| # CHECK-NOT: S_WAITCNT |
| |
| # CHECK: bb.1: |
| # CHECK-NEXT: V_LSHLREV_B64_e64 |
| # CHECK-NEXT: S_WAITCNT 112 |
| # CHECK-NEXT: FLAT_STORE_DWORD |
| name: single_fallthrough_successor_no_end_block_wait |
| |
| body: | |
| bb.0: |
| successors: %bb.1 |
| $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr |
| |
| bb.1: |
| $vgpr3_vgpr4 = V_LSHLREV_B64_e64 4, $vgpr7_vgpr8, implicit $exec |
| FLAT_STORE_DWORD $vgpr3_vgpr4, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr |
| S_ENDPGM 0 |
| ... |
| --- |
| # The block has a single predecessor with a single successor, but it |
| # is not the next block so it's non-obvious that the wait is not needed. |
| |
| |
| # CHECK-LABEL: name: single_branch_successor_not_next_block |
| |
| # CHECK: bb.1 |
| # CHECK-NEXT: FLAT_STORE_DWORD |
| # CHECK-NEXT: S_ENDPGM 0 |
| |
| # CHECK: bb.2: |
| # CHECK-NEXT: V_LSHLREV_B64_e64 |
| # CHECK-NEXT: S_WAITCNT 112 |
| # CHECK-NEXT: FLAT_STORE_DWORD |
| name: single_branch_successor_not_next_block |
| |
| body: | |
| bb.0: |
| successors: %bb.2 |
| $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr |
| S_BRANCH %bb.2 |
| |
| bb.1: |
| FLAT_STORE_DWORD $vgpr8_vgpr9, $vgpr10, 0, 0, implicit $exec, implicit $flat_scr |
| S_ENDPGM 0 |
| |
| bb.2: |
| $vgpr3_vgpr4 = V_LSHLREV_B64_e64 4, $vgpr7_vgpr8, implicit $exec |
| FLAT_STORE_DWORD $vgpr3_vgpr4, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr |
| S_ENDPGM 0 |
| ... |
| |
| # CHECK-LABEL: name: preexisting_waitcnt{{$}} |
| # CHECK: FLAT_LOAD_DWORD |
| # CHECK-NEXT: S_WAITCNT 0 |
| # CHECK-NOT: S_WAITCNT |
| name: preexisting_waitcnt |
| tracksRegLiveness: true |
| machineFunctionInfo: |
| isEntryFunction: true |
| body: | |
| bb.0: |
| liveins: $vgpr1_vgpr2 |
| $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr |
| S_WAITCNT 0 |
| FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr |
| |
| ... |
| |
| --- |
| |
| # CHECK-LABEL: name: bundle_no_waitcnt{{$}} |
| # CHECK: FLAT_LOAD_DWORD |
| # CHECK-NEXT: BUNDLE |
| # CHECK-NEXT: S_NOP |
| # CHECK-NEXT: S_NOP |
| # CHECK-NEXT: } |
| # CHECK-NEXT: S_WAITCNT 112 |
| name: bundle_no_waitcnt |
| tracksRegLiveness: true |
| machineFunctionInfo: |
| isEntryFunction: true |
| body: | |
| bb.0: |
| liveins: $vgpr1_vgpr2 |
| $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr |
| BUNDLE { |
| S_NOP 0 |
| S_NOP 0 |
| } |
| FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr |
| |
| ... |
| |
| --- |
| |
| # See the waitcnt inside the bundle and don't insert an extra |
| # CHECK-LABEL: name: preexisting_waitcnt_in_bundle{{$}} |
| # CHECK: FLAT_LOAD_DWORD |
| # CHECK: S_WAITCNT 0 |
| # CHECK-NOT: S_WAITCNT |
| name: preexisting_waitcnt_in_bundle |
| tracksRegLiveness: true |
| machineFunctionInfo: |
| isEntryFunction: true |
| body: | |
| bb.0: |
| liveins: $vgpr1_vgpr2 |
| $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr |
| BUNDLE { |
| S_NOP 0 |
| S_WAITCNT 0 |
| } |
| FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr |
| |
| ... |
| |
| --- |
| |
| # Def and use inside bundle |
| # CHECK-LABEL: name: insert_in_bundle{{$}} |
| # CHECK: BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 { |
| # CHECK-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr |
| # CHECK-NEXT: S_WAITCNT 112 |
| # CHECK-NEXT: FLAT_STORE_DWORD $vgpr1_vgpr2, internal $vgpr0, 0, 0, implicit $exec, implicit $flat_scr |
| # CHECK-NEXT: } |
| |
| name: insert_in_bundle |
| tracksRegLiveness: true |
| machineFunctionInfo: |
| isEntryFunction: true |
| body: | |
| bb.0: |
| liveins: $vgpr1_vgpr2 |
| BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 { |
| $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr |
| FLAT_STORE_DWORD $vgpr1_vgpr2, internal $vgpr0, 0, 0, implicit $exec, implicit $flat_scr |
| } |
| ... |
| |
| --- |
| |
| # Def is last instruction in bundle, use is outside bundle |
| |
| # CHECK-LABEL: name: exit_bundle{{$}} |
| # CHECK: BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 { |
| # CHECK-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr |
| # CHECK-NEXT: } |
| # CHECK-NEXT: S_WAITCNT 112 |
| # CHECK-NEXT: FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr |
| |
| name: exit_bundle |
| tracksRegLiveness: true |
| machineFunctionInfo: |
| isEntryFunction: true |
| body: | |
| bb.0: |
| liveins: $vgpr1_vgpr2 |
| BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 { |
| $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr |
| } |
| |
| FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr |
| |
| ... |
| |
| --- |
| |
| # Def is in bundle, use is in another bundle |
| |
| # CHECK-LABEL: name: cross_bundle{{$}} |
| # CHECK: BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 { |
| # CHECK-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr |
| # CHECK-NEXT: } |
| # CHECK-NEXT: S_WAITCNT 112 |
| # CHECK-NEXT: BUNDLE implicit $vgpr0, implicit $vgpr1_vgpr2 { |
| # CHECK-NEXT: FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr |
| # CHECK-NEXT: } |
| |
| name: cross_bundle |
| tracksRegLiveness: true |
| machineFunctionInfo: |
| isEntryFunction: true |
| body: | |
| bb.0: |
| liveins: $vgpr1_vgpr2 |
| BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 { |
| $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, implicit $exec, implicit $flat_scr |
| } |
| BUNDLE implicit $vgpr0, implicit $vgpr1_vgpr2 { |
| FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr |
| } |
| ... |
| |
| --- |
| # CHECK-LABEL: name: subregs16bit |
| # CHECK: S_WAITCNT 112 |
| # CHECK-NEXT: V_NOP_e32 |
| |
| name: subregs16bit |
| machineFunctionInfo: |
| isEntryFunction: true |
| body: | |
| bb.0: |
| liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4 |
| $vgpr0 = FLAT_LOAD_USHORT killed $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr |
| $vgpr1 = FLAT_LOAD_USHORT killed $vgpr2_vgpr3, 0, 0, implicit $exec, implicit $flat_scr |
| V_NOP_e32 implicit $exec, implicit $vgpr0_lo16, implicit $vgpr1_lo16 |
| ... |