blob: 029748d15077f22498447de092cb4d7fb3769770 [file] [edit]
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
# RUN: llc -mtriple=amdgpu12.50 -run-pass=si-insert-waitcnts,post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GFX1250 %s
# RUN: llc -mtriple=amdgpu12.51 -run-pass=si-insert-waitcnts,post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GFX1250 %s
# RUN: llc -mtriple=amdgpu13.10 -run-pass=si-insert-waitcnts,post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GFX1310 %s
# V_PERM_PK16 must be immediately followed by a "safe" instruction, which clears
# the hazard. A follower is safe iff it issues on the VALU pipe and belongs to
# one of the safe HW SP op-type buckets: OP_XDL, OP_32_T (genuine
# transcendentals), OP16_1, OP32_1, OP_CMACC, or OP_DUAL_1. The unsafe buckets
# are the multi-pass ops (GFX1250 BlockingCycles >= 2): FP64, 64-bit integer
# ops, I32_MUL (V_MUL_LO/HI_U32), QSAD/MQSAD, CVT_SCALE_PK*, and V_PERM_PK16
# itself, plus the pseudo-scalar transcendentals (OP32_SCL_T). When no safe
# instruction follows, a V_NOP is inserted. EXEC is already non-zero: V_PERM_PK16
# has unwanted effects when EXEC is empty, so s_cbranch_execz over the region is
# retained and a V_NOP does not degenerate to S_NOP.
#
# The cross-wave entry-block mitigation is delegated to the mandatory
# unclaused-VMEM entry prologue (GLOBAL_PREFETCH_B8 + V_NOP) emitted by
# SIInsertWaitcnts on every entry function, so this test runs that pass before
# the hazard recognizer to exercise the full sequence.
#
# The hazard only affects the GFX12 (gfx1250) part. gfx13 (gfx1310) has the same
# instructions but issues them safely, so it serves as negative coverage: no
# mitigation is inserted in any of the cases below.
# OP32_1 / OP_CMACC (V_MOV_B32) is safe: no mitigation.
---
name: perm_then_mov_safe
machineFunctionInfo:
isEntryFunction: false
sgprForEXECCopy: '$sgpr0'
body: |
bb.0:
; GFX1250-LABEL: name: perm_then_mov_safe
; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1250-NEXT: S_WAIT_KMCNT 0
; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1250-NEXT: $vgpr4 = V_MOV_B32_e32 0, implicit $exec
;
; GFX1310-LABEL: name: perm_then_mov_safe
; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1310-NEXT: S_WAIT_EXPCNT 0
; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
; GFX1310-NEXT: S_WAIT_BVHCNT 0
; GFX1310-NEXT: S_WAIT_KMCNT 0
; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1310-NEXT: $vgpr4 = V_MOV_B32_e32 0, implicit $exec
$vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
$vgpr4 = V_MOV_B32_e32 0, implicit $exec
...
# A packed VOP3P op (OP32_1) is safe: no mitigation.
---
name: perm_then_packed_safe
machineFunctionInfo:
isEntryFunction: false
sgprForEXECCopy: '$sgpr0'
body: |
bb.0:
; GFX1250-LABEL: name: perm_then_packed_safe
; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1250-NEXT: S_WAIT_KMCNT 0
; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1250-NEXT: $vgpr4 = V_PK_ADD_U16 8, $vgpr10, 8, $vgpr11, 0, 0, 0, 0, 0, implicit $exec
;
; GFX1310-LABEL: name: perm_then_packed_safe
; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1310-NEXT: S_WAIT_EXPCNT 0
; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
; GFX1310-NEXT: S_WAIT_BVHCNT 0
; GFX1310-NEXT: S_WAIT_KMCNT 0
; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1310-NEXT: $vgpr4 = V_PK_ADD_U16 8, $vgpr10, 8, $vgpr11, 0, 0, 0, 0, 0, implicit $exec
$vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
$vgpr4 = V_PK_ADD_U16 8, $vgpr10, 8, $vgpr11, 0, 0, 0, 0, 0, implicit $exec
...
# A 16-bit op (OP_CMACC/OP16_1, V_ADD_F16) is safe: no mitigation.
---
name: perm_then_op16_safe
machineFunctionInfo:
isEntryFunction: false
sgprForEXECCopy: '$sgpr0'
body: |
bb.0:
; GFX1250-LABEL: name: perm_then_op16_safe
; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1250-NEXT: S_WAIT_KMCNT 0
; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1250-NEXT: $vgpr4 = V_ADD_F16_fake16_e64 0, $vgpr10, 0, $vgpr11, 0, 0, implicit $mode, implicit $exec
;
; GFX1310-LABEL: name: perm_then_op16_safe
; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1310-NEXT: S_WAIT_EXPCNT 0
; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
; GFX1310-NEXT: S_WAIT_BVHCNT 0
; GFX1310-NEXT: S_WAIT_KMCNT 0
; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1310-NEXT: $vgpr4 = V_ADD_F16_fake16_e64 0, $vgpr10, 0, $vgpr11, 0, 0, implicit $mode, implicit $exec
$vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
$vgpr4 = V_ADD_F16_fake16_e64 0, $vgpr10, 0, $vgpr11, 0, 0, implicit $mode, implicit $exec
...
# A scalar FMA-accumulate op (OP_CMACC, V_FMAC_F32) is safe: no mitigation.
---
name: perm_then_cmacc_safe
machineFunctionInfo:
isEntryFunction: false
sgprForEXECCopy: '$sgpr0'
body: |
bb.0:
; GFX1250-LABEL: name: perm_then_cmacc_safe
; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1250-NEXT: S_WAIT_KMCNT 0
; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1250-NEXT: $vgpr4 = V_FMAC_F32_e64 0, $vgpr10, 0, $vgpr11, 0, $vgpr4, 0, 0, implicit $mode, implicit $exec
;
; GFX1310-LABEL: name: perm_then_cmacc_safe
; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1310-NEXT: S_WAIT_EXPCNT 0
; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
; GFX1310-NEXT: S_WAIT_BVHCNT 0
; GFX1310-NEXT: S_WAIT_KMCNT 0
; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1310-NEXT: $vgpr4 = V_FMAC_F32_e64 0, $vgpr10, 0, $vgpr11, 0, $vgpr4, 0, 0, implicit $mode, implicit $exec
$vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
$vgpr4 = V_FMAC_F32_e64 0, $vgpr10, 0, $vgpr11, 0, $vgpr4, 0, 0, implicit $mode, implicit $exec
...
# A lane/control op (OP32_1, V_READFIRSTLANE_B32) is safe: no mitigation.
---
name: perm_then_lane_safe
machineFunctionInfo:
isEntryFunction: false
sgprForEXECCopy: '$sgpr0'
body: |
bb.0:
; GFX1250-LABEL: name: perm_then_lane_safe
; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1250-NEXT: S_WAIT_KMCNT 0
; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1250-NEXT: $sgpr4 = V_READFIRSTLANE_B32 $vgpr10, implicit $exec
;
; GFX1310-LABEL: name: perm_then_lane_safe
; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1310-NEXT: S_WAIT_EXPCNT 0
; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
; GFX1310-NEXT: S_WAIT_BVHCNT 0
; GFX1310-NEXT: S_WAIT_KMCNT 0
; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1310-NEXT: $sgpr4 = V_READFIRSTLANE_B32 $vgpr10, implicit $exec
$vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
$sgpr4 = V_READFIRSTLANE_B32 $vgpr10, implicit $exec
...
# A genuine TRANS instruction (OP_32_T, V_RCP_F32) is safe: no mitigation.
---
name: perm_then_trans_safe
machineFunctionInfo:
isEntryFunction: false
sgprForEXECCopy: '$sgpr0'
body: |
bb.0:
; GFX1250-LABEL: name: perm_then_trans_safe
; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1250-NEXT: S_WAIT_KMCNT 0
; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1250-NEXT: $vgpr4 = V_RCP_F32_e32 $vgpr10, implicit $mode, implicit $exec
;
; GFX1310-LABEL: name: perm_then_trans_safe
; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1310-NEXT: S_WAIT_EXPCNT 0
; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
; GFX1310-NEXT: S_WAIT_BVHCNT 0
; GFX1310-NEXT: S_WAIT_KMCNT 0
; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1310-NEXT: $vgpr4 = V_RCP_F32_e32 $vgpr10, implicit $mode, implicit $exec
$vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
$vgpr4 = V_RCP_F32_e32 $vgpr10, implicit $mode, implicit $exec
...
# A multi-pass op (I32_MUL, V_MUL_LO_U32) is NOT safe: insert a V_NOP.
---
name: perm_then_i32mul_nonsafe
machineFunctionInfo:
isEntryFunction: false
sgprForEXECCopy: '$sgpr0'
body: |
bb.0:
; GFX1250-LABEL: name: perm_then_i32mul_nonsafe
; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1250-NEXT: S_WAIT_KMCNT 0
; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1250-NEXT: V_NOP_e32 implicit $exec
; GFX1250-NEXT: $vgpr4 = V_MUL_LO_U32_e64 $vgpr10, $vgpr11, implicit $exec
;
; GFX1310-LABEL: name: perm_then_i32mul_nonsafe
; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1310-NEXT: S_WAIT_EXPCNT 0
; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
; GFX1310-NEXT: S_WAIT_BVHCNT 0
; GFX1310-NEXT: S_WAIT_KMCNT 0
; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1310-NEXT: $vgpr4 = V_MUL_LO_U32_e64 $vgpr10, $vgpr11, implicit $exec
$vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
$vgpr4 = V_MUL_LO_U32_e64 $vgpr10, $vgpr11, implicit $exec
...
# A multi-pass op (QSAD, V_MQSAD_PK_U16_U8 - the post-silicon bug case) is NOT
# safe: insert a V_NOP.
---
name: perm_then_qsad_nonsafe
machineFunctionInfo:
isEntryFunction: false
sgprForEXECCopy: '$sgpr0'
body: |
bb.0:
; GFX1250-LABEL: name: perm_then_qsad_nonsafe
; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1250-NEXT: S_WAIT_KMCNT 0
; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1250-NEXT: V_NOP_e32 implicit $exec
; GFX1250-NEXT: early-clobber $vgpr4_vgpr5 = V_MQSAD_PK_U16_U8_e64 $vgpr10_vgpr11, $vgpr12, $vgpr14_vgpr15, 0, implicit $exec
;
; GFX1310-LABEL: name: perm_then_qsad_nonsafe
; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1310-NEXT: S_WAIT_EXPCNT 0
; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
; GFX1310-NEXT: S_WAIT_BVHCNT 0
; GFX1310-NEXT: S_WAIT_KMCNT 0
; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1310-NEXT: early-clobber $vgpr4_vgpr5 = V_MQSAD_PK_U16_U8_e64 $vgpr10_vgpr11, $vgpr12, $vgpr14_vgpr15, 0, implicit $exec
$vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
$vgpr4_vgpr5 = V_MQSAD_PK_U16_U8_e64 $vgpr10_vgpr11, $vgpr12, $vgpr14_vgpr15, 0, implicit $exec
...
# An F64 transcendental is NOT safe: insert a V_NOP.
---
name: perm_then_f64_trans
machineFunctionInfo:
isEntryFunction: false
sgprForEXECCopy: '$sgpr0'
body: |
bb.0:
; GFX1250-LABEL: name: perm_then_f64_trans
; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1250-NEXT: S_WAIT_KMCNT 0
; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1250-NEXT: V_NOP_e32 implicit $exec
; GFX1250-NEXT: $vgpr4_vgpr5 = V_RCP_F64_e32 $vgpr10_vgpr11, implicit $mode, implicit $exec
;
; GFX1310-LABEL: name: perm_then_f64_trans
; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1310-NEXT: S_WAIT_EXPCNT 0
; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
; GFX1310-NEXT: S_WAIT_BVHCNT 0
; GFX1310-NEXT: S_WAIT_KMCNT 0
; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1310-NEXT: $vgpr4_vgpr5 = V_RCP_F64_e32 $vgpr10_vgpr11, implicit $mode, implicit $exec
$vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
$vgpr4_vgpr5 = V_RCP_F64_e32 $vgpr10_vgpr11, implicit $mode, implicit $exec
...
# A pseudo-scalar transcendental (OP32_SCL_T, V_S_EXP_F32) is NOT safe: insert
# a V_NOP.
---
name: perm_then_pseudo_scalar_trans_nonsafe
machineFunctionInfo:
isEntryFunction: false
sgprForEXECCopy: '$sgpr0'
body: |
bb.0:
; GFX1250-LABEL: name: perm_then_pseudo_scalar_trans_nonsafe
; GFX1250: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1250-NEXT: S_WAIT_KMCNT 0
; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1250-NEXT: V_NOP_e32 implicit $exec
; GFX1250-NEXT: $sgpr4 = V_S_EXP_F32_e64 0, $sgpr10, 0, 0, implicit $mode, implicit $exec
;
; GFX1310-LABEL: name: perm_then_pseudo_scalar_trans_nonsafe
; GFX1310: S_WAIT_LOADCNT_DSCNT .Loadcnt_0_Dscnt_0
; GFX1310-NEXT: S_WAIT_EXPCNT 0
; GFX1310-NEXT: S_WAIT_SAMPLECNT 0
; GFX1310-NEXT: S_WAIT_BVHCNT 0
; GFX1310-NEXT: S_WAIT_KMCNT 0
; GFX1310-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1310-NEXT: $sgpr4 = V_S_EXP_F32_e64 0, $sgpr10, 0, 0, implicit $mode, implicit $exec
$vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
$sgpr4 = V_S_EXP_F32_e64 0, $sgpr10, 0, 0, implicit $mode, implicit $exec
...
# Entry function: defensive mitigation at the very start plus the per-perm one.
---
name: perm_kernel_entry
machineFunctionInfo:
isEntryFunction: true
sgprForEXECCopy: '$sgpr0'
body: |
bb.0:
; GFX1250-LABEL: name: perm_kernel_entry
; GFX1250: $sgpr64_sgpr65 = S_MOV_B64 0
; GFX1250-NEXT: V_NOP_e32 implicit $exec
; GFX1250-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
; GFX1250-NEXT: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1250-NEXT: V_NOP_e32 implicit $exec
; GFX1250-NEXT: S_ENDPGM 0
;
; GFX1310-LABEL: name: perm_kernel_entry
; GFX1310: $vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
; GFX1310-NEXT: S_ENDPGM 0
$vgpr0_vgpr1 = V_PERM_PK16_B4_U4_e64 $vgpr0, $vgpr1, $vgpr2_vgpr3, implicit $exec
S_ENDPGM 0
...
# Entry function with no V_PERM_PK16 at all: the defensive entry-block mitigation
# is still inserted. Minimal mitigation for the potentially cross-wave hazard.
---
name: no_perm_kernel_entry
machineFunctionInfo:
isEntryFunction: true
sgprForEXECCopy: '$sgpr0'
body: |
bb.0:
; GFX1250-LABEL: name: no_perm_kernel_entry
; GFX1250: $sgpr64_sgpr65 = S_MOV_B64 0
; GFX1250-NEXT: V_NOP_e32 implicit $exec
; GFX1250-NEXT: GLOBAL_PREFETCH_B8_SADDR $sgpr64_sgpr65, undef $vgpr0, 0, 8, implicit $exec
; GFX1250-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX1250-NEXT: S_ENDPGM 0
;
; GFX1310-LABEL: name: no_perm_kernel_entry
; GFX1310: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX1310-NEXT: S_ENDPGM 0
$vgpr0 = V_MOV_B32_e32 0, implicit $exec
S_ENDPGM 0
...