Google Git
Sign in
llvm/llvm-project.git/refs/heads/main/./llvm/test/CodeGen/AMDGPU/coexec-hazardrec-preRA.mir
blob: b4e8a86de06b8e76e86869698a7268b667479357 [file] [edit]
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -verify-misched %s -o - | FileCheck -check-prefix=CHECK %s
# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -verify-misched -debug-only=gcn-hazard-recognizer %s -o /dev/null 2>&1 | FileCheck -check-prefix=DBG %s
# REQUIRES: asserts
# Tests use V_WMMA_SCALE_F32_16X16X128_F8F6F4 which has pattern 0EEIEEISVV
# (10-cycle window, 8-cycle occupancy). The 'S' slot indicates scaled WMMA
# can issue there (LD_SCALE rule).
#
# Co-execution window slots:
# Stage: 0 1 2 3 4 5 6 7 8 9
# Slot: E0 E E I E E I S V V
#
# E0 (stage 0): Control only (issue cycle)
# E (stages 1,2,4,5): MEM + SALU allowed, NO VALU/TRANS
# I (stages 3,6): MEM + SALU + VALU + TRANS allowed
# S (stage 7): Same as I, plus scaled WMMA can absorb here
# V (stages 8,9): MEM + SALU + WMMA allowed, NO VALU/TRANS
--- |
define void @wmma_ds_salu_only() #0 { ret void }
define void @wmma_valu_trans_only() #0 { ret void }
define void @wmma_valu_in_I_slots() #0 { ret void }
define void @wmma_trans_in_I_slots() #0 { ret void }
define void @wmma_chain_back_to_back() #0 { ret void }
define void @wmma_mixed_ds_valu_trans() #0 { ret void }
define void @trans_valu_interleave() #0 { ret void }
define void @wmma_valu_blocked_in_V_slot() #0 { ret void }
attributes #0 = { "amdgpu-waves-per-eu"="1,1" }
...
# DBG-LABEL: PreRA hazard recognizer: wmma_ds_salu_only
# DBG: CoExec window complete:
# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
# DBG-NEXT: Slots: 0 E E I E E I S V V
# DBG-NEXT: Scheduled: 0 E E I E E I S V V
---
name: wmma_ds_salu_only
tracksRegLiveness: true
body: |
; CHECK-LABEL: name: wmma_ds_salu_only
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF7:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: early-clobber %17:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DEF3]], [[DEF4]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 16, 0, implicit $exec
; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[DEF6]], [[DEF7]], implicit-def dead $scc
; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 32, 0, implicit $exec
; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_]], [[DEF7]], implicit-def dead $scc
; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 48, 0, implicit $exec
; CHECK-NEXT: [[S_ADD_I32_2:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_1]], [[DEF6]], implicit-def dead $scc
; CHECK-NEXT: [[DS_READ_B128_gfx9_4:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 64, 0, implicit $exec
; CHECK-NEXT: [[S_ADD_I32_3:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_2]], [[DEF7]], implicit-def dead $scc
; CHECK-NEXT: S_ENDPGM 0, implicit [[DS_READ_B128_gfx9_]], implicit [[DS_READ_B128_gfx9_1]], implicit [[DS_READ_B128_gfx9_2]], implicit [[DS_READ_B128_gfx9_3]], implicit [[DS_READ_B128_gfx9_4]], implicit [[S_ADD_I32_3]], implicit %17
bb.0:
successors: %bb.1
%0:vreg_512_align2 = IMPLICIT_DEF
%1:vreg_512_align2 = IMPLICIT_DEF
%2:vreg_256_align2 = IMPLICIT_DEF
%3:vgpr_32_lo256 = IMPLICIT_DEF
%4:vgpr_32_lo256 = IMPLICIT_DEF
%5:vgpr_32 = IMPLICIT_DEF
%6:sreg_32 = IMPLICIT_DEF
%7:sreg_32 = IMPLICIT_DEF
S_BRANCH %bb.1
bb.1:
%8:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 0, 0, implicit $exec
%9:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 16, 0, implicit $exec
%10:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 32, 0, implicit $exec
%11:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 48, 0, implicit $exec
%12:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 64, 0, implicit $exec
%13:sreg_32 = S_ADD_I32 %6, %7, implicit-def dead $scc
%14:sreg_32 = S_ADD_I32 %13, %7, implicit-def dead $scc
%15:sreg_32 = S_ADD_I32 %14, %6, implicit-def dead $scc
%16:sreg_32 = S_ADD_I32 %15, %7, implicit-def dead $scc
early-clobber %17:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
S_ENDPGM 0, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %16, implicit %17
...
# DBG-LABEL: PreRA hazard recognizer: wmma_valu_trans_only
# DBG: CoExec window ended at stage 7:
# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
# DBG-NEXT: Slots: 0 E E I E E I S V V
# DBG-NEXT: Scheduled: 0 - - I - - I S . .
---
name: wmma_valu_trans_only
tracksRegLiveness: true
body: |
; CHECK-LABEL: name: wmma_valu_trans_only
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF6:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF7:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: early-clobber %10:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DEF3]], [[DEF4]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; CHECK-NEXT: [[V_EXP_F32_e32_:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF5]], implicit $mode, implicit $exec
; CHECK-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF6]], 8, [[DEF7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_EXP_F32_e32_]], implicit [[V_PK_ADD_F32_]], implicit %10
bb.0:
successors: %bb.1
%0:vreg_512_align2 = IMPLICIT_DEF
%1:vreg_512_align2 = IMPLICIT_DEF
%2:vreg_256_align2 = IMPLICIT_DEF
%3:vgpr_32_lo256 = IMPLICIT_DEF
%4:vgpr_32_lo256 = IMPLICIT_DEF
%5:vgpr_32 = IMPLICIT_DEF
%6:vreg_64_align2 = IMPLICIT_DEF
%7:vreg_64_align2 = IMPLICIT_DEF
S_BRANCH %bb.1
bb.1:
%8:vgpr_32 = V_EXP_F32_e32 %5, implicit $mode, implicit $exec
%9:vreg_64_align2 = V_PK_ADD_F32 8, %6, 8, %7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
early-clobber %10:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
S_ENDPGM 0, implicit %8, implicit %9, implicit %10
...
# DBG-LABEL: PreRA hazard recognizer: wmma_valu_in_I_slots
# DBG: CoExec window ended at stage 8:
# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
# DBG-NEXT: Slots: 0 E E I E E I S V V
# DBG-NEXT: Scheduled: 0 E E I E E I S V .
---
name: wmma_valu_in_I_slots
tracksRegLiveness: true
body: |
; CHECK-LABEL: name: wmma_valu_in_I_slots
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF6:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF7:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF8:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: early-clobber %16:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DEF3]], [[DEF4]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 16, 0, implicit $exec
; CHECK-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF6]], 8, [[DEF7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 32, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 48, 0, implicit $exec
; CHECK-NEXT: [[V_PK_ADD_F32_1:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF7]], 8, [[DEF8]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[V_PK_ADD_F32_2:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF6]], 8, [[DEF8]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[DS_READ_B128_gfx9_]], implicit [[DS_READ_B128_gfx9_1]], implicit [[DS_READ_B128_gfx9_2]], implicit [[DS_READ_B128_gfx9_3]], implicit [[V_PK_ADD_F32_]], implicit [[V_PK_ADD_F32_1]], implicit [[V_PK_ADD_F32_2]], implicit %16
bb.0:
successors: %bb.1
%0:vreg_512_align2 = IMPLICIT_DEF
%1:vreg_512_align2 = IMPLICIT_DEF
%2:vreg_256_align2 = IMPLICIT_DEF
%3:vgpr_32_lo256 = IMPLICIT_DEF
%4:vgpr_32_lo256 = IMPLICIT_DEF
%5:vgpr_32 = IMPLICIT_DEF
%6:vreg_64_align2 = IMPLICIT_DEF
%7:vreg_64_align2 = IMPLICIT_DEF
%8:vreg_64_align2 = IMPLICIT_DEF
S_BRANCH %bb.1
bb.1:
%9:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 0, 0, implicit $exec
%10:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 16, 0, implicit $exec
%11:vreg_64_align2 = V_PK_ADD_F32 8, %6, 8, %7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
%12:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 32, 0, implicit $exec
%13:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 48, 0, implicit $exec
%14:vreg_64_align2 = V_PK_ADD_F32 8, %7, 8, %8, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
%15:vreg_64_align2 = V_PK_ADD_F32 8, %6, 8, %8, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
early-clobber %16:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
S_ENDPGM 0, implicit %9, implicit %10, implicit %12, implicit %13, implicit %11, implicit %14, implicit %15, implicit %16
...
# DBG-LABEL: PreRA hazard recognizer: wmma_trans_in_I_slots
# DBG: CoExec window ended at stage 7:
# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
# DBG-NEXT: Slots: 0 E E I E E I S V V
# DBG-NEXT: Scheduled: 0 - - I - - I S . .
---
name: wmma_trans_in_I_slots
tracksRegLiveness: true
body: |
; CHECK-LABEL: name: wmma_trans_in_I_slots
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF6:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF7:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: early-clobber %10:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DEF3]], [[DEF4]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; CHECK-NEXT: [[V_EXP_F32_e32_:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF5]], implicit $mode, implicit $exec
; CHECK-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF6]], 8, [[DEF7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_EXP_F32_e32_]], implicit [[V_PK_ADD_F32_]], implicit %10
bb.0:
successors: %bb.1
%0:vreg_512_align2 = IMPLICIT_DEF
%1:vreg_512_align2 = IMPLICIT_DEF
%2:vreg_256_align2 = IMPLICIT_DEF
%3:vgpr_32_lo256 = IMPLICIT_DEF
%4:vgpr_32_lo256 = IMPLICIT_DEF
%5:vgpr_32 = IMPLICIT_DEF
%6:vreg_64_align2 = IMPLICIT_DEF
%7:vreg_64_align2 = IMPLICIT_DEF
S_BRANCH %bb.1
bb.1:
%8:vgpr_32 = V_EXP_F32_e32 %5, implicit $mode, implicit $exec
%9:vreg_64_align2 = V_PK_ADD_F32 8, %6, 8, %7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
early-clobber %10:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
S_ENDPGM 0, implicit %8, implicit %9, implicit %10
...
# DBG-LABEL: PreRA hazard recognizer: wmma_chain_back_to_back
# DBG: WMMA window started: 0EEIEEISVV (window=10)
# The next WMMA of the chain issues in a vacant slot, which starts a new window.
# DBG: CoExec window interrupted at stage 8:
# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
# DBG-NEXT: Slots: 0 E E I E E I S V V
# DBG-NEXT: Scheduled: 0 - - - - - - - V .
---
name: wmma_chain_back_to_back
tracksRegLiveness: true
body: |
; CHECK-LABEL: name: wmma_chain_back_to_back
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF3:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF6:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF7:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF8:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF9:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF10:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: early-clobber %17:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF6]], [[DEF7]], 0, [[DEF8]], [[DEF9]], [[DEF10]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; CHECK-NEXT: [[DEF11:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF12:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF13:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF14:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: early-clobber %15:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF11]], [[DEF12]], 0, [[DEF13]], [[DEF]], [[DEF1]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; CHECK-NEXT: early-clobber %16:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF14]], [[DEF2]], 0, [[DEF3]], [[DEF4]], [[DEF5]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit %15, implicit %16, implicit %17
bb.0:
successors: %bb.1
%0:vreg_512_align2 = IMPLICIT_DEF
%1:vreg_512_align2 = IMPLICIT_DEF
%2:vreg_256_align2 = IMPLICIT_DEF
%3:vgpr_32_lo256 = IMPLICIT_DEF
%4:vgpr_32_lo256 = IMPLICIT_DEF
%5:vreg_512_align2 = IMPLICIT_DEF
%6:vreg_512_align2 = IMPLICIT_DEF
%7:vreg_256_align2 = IMPLICIT_DEF
%8:vgpr_32_lo256 = IMPLICIT_DEF
%9:vgpr_32_lo256 = IMPLICIT_DEF
%10:vreg_512_align2 = IMPLICIT_DEF
%11:vreg_512_align2 = IMPLICIT_DEF
%12:vreg_256_align2 = IMPLICIT_DEF
%13:vgpr_32_lo256 = IMPLICIT_DEF
%14:vgpr_32_lo256 = IMPLICIT_DEF
S_BRANCH %bb.1
bb.1:
early-clobber %15:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
early-clobber %16:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %5, %6, 0, %7, %8, %9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
early-clobber %17:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %10, %11, 0, %12, %13, %14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
S_ENDPGM 0, implicit %15, implicit %16, implicit %17
...
# DBG-LABEL: PreRA hazard recognizer: wmma_mixed_ds_valu_trans
# DBG: CoExec window complete:
# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
# DBG-NEXT: Slots: 0 E E I E E I S V V
# DBG-NEXT: Scheduled: 0 E E I E E I S - -
---
name: wmma_mixed_ds_valu_trans
tracksRegLiveness: true
body: |
; CHECK-LABEL: name: wmma_mixed_ds_valu_trans
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF6:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF7:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF8:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF9:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF10:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF11:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: early-clobber %21:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DEF3]], [[DEF4]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 16, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 32, 0, implicit $exec
; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[DEF10]], [[DEF11]], implicit-def dead $scc
; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 48, 0, implicit $exec
; CHECK-NEXT: [[V_EXP_F32_e32_:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF8]], implicit $mode, implicit $exec
; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_ADD_I32_]], [[DEF10]], implicit-def dead $scc
; CHECK-NEXT: [[V_EXP_F32_e32_1:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF9]], implicit $mode, implicit $exec
; CHECK-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF6]], 8, [[DEF7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[DS_READ_B128_gfx9_]], implicit [[DS_READ_B128_gfx9_1]], implicit [[DS_READ_B128_gfx9_2]], implicit [[DS_READ_B128_gfx9_3]], implicit [[V_PK_ADD_F32_]], implicit [[V_EXP_F32_e32_]], implicit [[V_EXP_F32_e32_1]], implicit [[S_ADD_I32_1]], implicit %21
bb.0:
successors: %bb.1
%0:vreg_512_align2 = IMPLICIT_DEF
%1:vreg_512_align2 = IMPLICIT_DEF
%2:vreg_256_align2 = IMPLICIT_DEF
%3:vgpr_32_lo256 = IMPLICIT_DEF
%4:vgpr_32_lo256 = IMPLICIT_DEF
%5:vgpr_32 = IMPLICIT_DEF
%6:vreg_64_align2 = IMPLICIT_DEF
%7:vreg_64_align2 = IMPLICIT_DEF
%8:vgpr_32 = IMPLICIT_DEF
%9:vgpr_32 = IMPLICIT_DEF
%10:sreg_32 = IMPLICIT_DEF
%11:sreg_32 = IMPLICIT_DEF
S_BRANCH %bb.1
bb.1:
%12:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 0, 0, implicit $exec
%13:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 16, 0, implicit $exec
%14:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 32, 0, implicit $exec
%15:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 48, 0, implicit $exec
%16:vreg_64_align2 = V_PK_ADD_F32 8, %6, 8, %7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
%17:vgpr_32 = V_EXP_F32_e32 %8, implicit $mode, implicit $exec
%18:vgpr_32 = V_EXP_F32_e32 %9, implicit $mode, implicit $exec
%19:sreg_32 = S_ADD_I32 %10, %11, implicit-def dead $scc
%20:sreg_32 = S_ADD_I32 %19, %10, implicit-def dead $scc
early-clobber %21:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
S_ENDPGM 0, implicit %12, implicit %13, implicit %14, implicit %15, implicit %16, implicit %17, implicit %18, implicit %20, implicit %21
...
# TRANS does not create co-exec windows. The TRANS counter enforces
# 1-cycle gaps between TRANS instructions, interleaving them with VALU.
# DBG-LABEL: PreRA hazard recognizer: trans_valu_interleave
# DBG: TRANS hazard set: CyclesUntilTRANS=2
# DBG: TRANS hazard set: CyclesUntilTRANS=2
# DBG: TRANS hazard set: CyclesUntilTRANS=2
# DBG: TRANS hazard set: CyclesUntilTRANS=2
---
name: trans_valu_interleave
tracksRegLiveness: true
body: |
; CHECK-LABEL: name: trans_valu_interleave
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF6:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF7:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: [[V_EXP_F32_e32_:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF]], implicit $mode, implicit $exec
; CHECK-NEXT: [[V_ADD_F32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_F32_e32 [[DEF4]], [[DEF5]], implicit $mode, implicit $exec
; CHECK-NEXT: [[V_EXP_F32_e32_1:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF1]], implicit $mode, implicit $exec
; CHECK-NEXT: [[V_ADD_F32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_F32_e32 [[DEF5]], [[DEF6]], implicit $mode, implicit $exec
; CHECK-NEXT: [[V_EXP_F32_e32_2:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF2]], implicit $mode, implicit $exec
; CHECK-NEXT: [[V_ADD_F32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_F32_e32 [[DEF6]], [[DEF7]], implicit $mode, implicit $exec
; CHECK-NEXT: [[V_EXP_F32_e32_3:%[0-9]+]]:vgpr_32 = V_EXP_F32_e32 [[DEF3]], implicit $mode, implicit $exec
; CHECK-NEXT: [[V_ADD_F32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_F32_e32 [[DEF7]], [[DEF4]], implicit $mode, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[V_EXP_F32_e32_]], implicit [[V_EXP_F32_e32_1]], implicit [[V_EXP_F32_e32_2]], implicit [[V_EXP_F32_e32_3]], implicit [[V_ADD_F32_e32_]], implicit [[V_ADD_F32_e32_1]], implicit [[V_ADD_F32_e32_2]], implicit [[V_ADD_F32_e32_3]]
bb.0:
successors: %bb.1
%0:vgpr_32 = IMPLICIT_DEF
%1:vgpr_32 = IMPLICIT_DEF
%2:vgpr_32 = IMPLICIT_DEF
%3:vgpr_32 = IMPLICIT_DEF
%4:vgpr_32 = IMPLICIT_DEF
%5:vgpr_32 = IMPLICIT_DEF
%6:vgpr_32 = IMPLICIT_DEF
%7:vgpr_32 = IMPLICIT_DEF
S_BRANCH %bb.1
bb.1:
%8:vgpr_32 = V_EXP_F32_e32 %0, implicit $mode, implicit $exec
%9:vgpr_32 = V_EXP_F32_e32 %1, implicit $mode, implicit $exec
%10:vgpr_32 = V_EXP_F32_e32 %2, implicit $mode, implicit $exec
%11:vgpr_32 = V_EXP_F32_e32 %3, implicit $mode, implicit $exec
%12:vgpr_32 = V_ADD_F32_e32 %4, %5, implicit $mode, implicit $exec
%13:vgpr_32 = V_ADD_F32_e32 %5, %6, implicit $mode, implicit $exec
%14:vgpr_32 = V_ADD_F32_e32 %6, %7, implicit $mode, implicit $exec
%15:vgpr_32 = V_ADD_F32_e32 %7, %4, implicit $mode, implicit $exec
S_ENDPGM 0, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14, implicit %15
...
# Test 8: VALU blocked in V slots. Fill E and I slots with DS/VALU,
# then extra VALU that cannot go in V slots (stages 8,9).
# Only 2 I slots available (3,6) but 5 VALU - some VALU pushed past window.
# DBG-LABEL: PreRA hazard recognizer: wmma_valu_blocked_in_V_slot
# DBG: CoExec window complete:
# DBG-NEXT: Stages: 0 1 2 3 4 5 6 7 8 9
# DBG-NEXT: Slots: 0 E E I E E I S V V
# DBG-NEXT: Scheduled: 0 E E I E - I S - -
---
name: wmma_valu_blocked_in_V_slot
tracksRegLiveness: true
body: |
; CHECK-LABEL: name: wmma_valu_blocked_in_V_slot
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32_lo256 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF6:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF7:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF8:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: [[DEF9:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: early-clobber %19:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DEF3]], [[DEF4]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 0, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_1:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 16, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_2:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 32, 0, implicit $exec
; CHECK-NEXT: [[DS_READ_B128_gfx9_3:%[0-9]+]]:vreg_128_lo256_align2 = DS_READ_B128_gfx9 [[DEF5]], 48, 0, implicit $exec
; CHECK-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF6]], 8, [[DEF7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[V_PK_ADD_F32_1:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF7]], 8, [[DEF8]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[V_PK_ADD_F32_2:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF8]], 8, [[DEF9]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[V_PK_ADD_F32_3:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF6]], 8, [[DEF9]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[V_PK_ADD_F32_4:%[0-9]+]]:vreg_64_align2 = V_PK_ADD_F32 8, [[DEF9]], 8, [[DEF7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_ENDPGM 0, implicit [[DS_READ_B128_gfx9_]], implicit [[DS_READ_B128_gfx9_1]], implicit [[DS_READ_B128_gfx9_2]], implicit [[DS_READ_B128_gfx9_3]], implicit [[V_PK_ADD_F32_]], implicit [[V_PK_ADD_F32_1]], implicit [[V_PK_ADD_F32_2]], implicit [[V_PK_ADD_F32_3]], implicit [[V_PK_ADD_F32_4]], implicit %19
bb.0:
successors: %bb.1
%0:vreg_512_align2 = IMPLICIT_DEF
%1:vreg_512_align2 = IMPLICIT_DEF
%2:vreg_256_align2 = IMPLICIT_DEF
%3:vgpr_32_lo256 = IMPLICIT_DEF
%4:vgpr_32_lo256 = IMPLICIT_DEF
%5:vgpr_32 = IMPLICIT_DEF
%6:vreg_64_align2 = IMPLICIT_DEF
%7:vreg_64_align2 = IMPLICIT_DEF
%8:vreg_64_align2 = IMPLICIT_DEF
%9:vreg_64_align2 = IMPLICIT_DEF
S_BRANCH %bb.1
bb.1:
%10:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 0, 0, implicit $exec
%11:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 16, 0, implicit $exec
%12:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 32, 0, implicit $exec
%13:vreg_128_lo256_align2 = DS_READ_B128_gfx9 %5, 48, 0, implicit $exec
%14:vreg_64_align2 = V_PK_ADD_F32 8, %6, 8, %7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
%15:vreg_64_align2 = V_PK_ADD_F32 8, %7, 8, %8, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
%16:vreg_64_align2 = V_PK_ADD_F32 8, %8, 8, %9, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
%17:vreg_64_align2 = V_PK_ADD_F32 8, %6, 8, %9, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
%18:vreg_64_align2 = V_PK_ADD_F32 8, %9, 8, %7, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
early-clobber %19:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %0, %1, 0, %2, %3, %4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
S_ENDPGM 0, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14, implicit %15, implicit %16, implicit %17, implicit %18, implicit %19
...