blob: b12bdaec39191768c3ab2d0feb5cb2850f3e80ca [file] [edit]
; REQUIRES: asserts
; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -amdgpu-avgpr-banking=1 \
; RUN: -debug-only=machine-scheduler %s -o - 2>&1 \
; RUN: | FileCheck %s -check-prefixes=BANK,NOSPILL \
; RUN: -implicit-check-not="Region is known to spill" \
; RUN: -implicit-check-not="Reverting scheduling"
; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -amdgpu-avgpr-banking=0 \
; RUN: -debug-only=machine-scheduler %s -o - 2>&1 \
; RUN: | FileCheck %s -check-prefixes=NOBANK,NOSPILL
; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -amdgpu-use-amdgpu-trackers=1 \
; RUN: -amdgpu-avgpr-banking=1 -debug-only=machine-scheduler %s -o - 2>&1 \
; RUN: | FileCheck %s -check-prefixes=BANK,NOSPILL \
; RUN: -implicit-check-not="Region is known to spill" \
; RUN: -implicit-check-not="Reverting scheduling"
; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -amdgpu-use-amdgpu-trackers=1 \
; RUN: -amdgpu-avgpr-banking=0 -debug-only=machine-scheduler %s -o - 2>&1 \
; RUN: | FileCheck %s -check-prefixes=NOBANK,NOSPILL
; Nearly all of this kernel's vector pressure sits in av_* values, which the
; register allocator may place in either the ArchVGPR or the AGPR bank.
;
; The IR is shared with coexec-rewrite-mfma.ll, reduced there from a
; Triton-generated GEMM with 30 MFMAs in a phi-carried loop.
; BANK: Region register pressure: VGPRs: {{[0-9]+}} (AV: {{[0-9]+}}) AGPRs: {{[0-9]+}}
; NOBANK: Region register pressure: VGPRs: {{[0-9]+}} (AV: {{[0-9]+}}) AGPRs: {{[0-9]+}}
; The two stages report these in either order depending on the tracker in use.
; NOBANK-DAG: Region is known to spill
; NOBANK-DAG: Reverting scheduling for region
; The checks above are what the scheduler predicts. Neither configuration
; spills in the end.
; NOSPILL: ScratchSize: 0
define amdgpu_kernel void @mfma_loop_high_av_pressure() #0 {
entry:
%i = load <8 x half>, ptr addrspace(3) null, align 16
%i1 = shufflevector <8 x half> %i, <8 x half> zeroinitializer, <2 x i32> <i32 0, i32 1>
%i2 = shufflevector <8 x half> %i, <8 x half> zeroinitializer, <2 x i32> <i32 2, i32 3>
br label %loop
loop: ; preds = %loop, %entry
%i3 = phi float [ 0.000000e+00, %entry ], [ %i52, %loop ]
%i4 = phi float [ 0.000000e+00, %entry ], [ %i53, %loop ]
%i5 = phi float [ 0.000000e+00, %entry ], [ %i60, %loop ]
%i6 = phi float [ 0.000000e+00, %entry ], [ %i61, %loop ]
%i7 = phi float [ 0.000000e+00, %entry ], [ %i62, %loop ]
%i8 = phi float [ 0.000000e+00, %entry ], [ %i63, %loop ]
%i9 = phi float [ 0.000000e+00, %entry ], [ %i70, %loop ]
%i10 = phi float [ 0.000000e+00, %entry ], [ %i71, %loop ]
%i11 = phi float [ 0.000000e+00, %entry ], [ %i72, %loop ]
%i12 = phi float [ 0.000000e+00, %entry ], [ %i73, %loop ]
%i13 = phi float [ 0.000000e+00, %entry ], [ %i79, %loop ]
%i14 = phi float [ 0.000000e+00, %entry ], [ %i80, %loop ]
%i15 = phi float [ 0.000000e+00, %entry ], [ %i81, %loop ]
%i16 = phi float [ 0.000000e+00, %entry ], [ %i86, %loop ]
%i17 = phi float [ 0.000000e+00, %entry ], [ %i87, %loop ]
%i18 = phi float [ 0.000000e+00, %entry ], [ %i92, %loop ]
%i19 = phi float [ 0.000000e+00, %entry ], [ %i93, %loop ]
%i20 = phi float [ 0.000000e+00, %entry ], [ %i99, %loop ]
%i21 = phi float [ 0.000000e+00, %entry ], [ %i100, %loop ]
%i22 = phi float [ 0.000000e+00, %entry ], [ %i23, %loop ]
%i23 = phi float [ 0.000000e+00, %entry ], [ %i105, %loop ]
%i24 = phi float [ 0.000000e+00, %entry ], [ %i112, %loop ]
%i25 = phi float [ 0.000000e+00, %entry ], [ %i113, %loop ]
%i26 = phi float [ 0.000000e+00, %entry ], [ %i114, %loop ]
%i27 = phi float [ 0.000000e+00, %entry ], [ %i115, %loop ]
%i28 = phi float [ 0.000000e+00, %entry ], [ %i122, %loop ]
%i29 = phi float [ 0.000000e+00, %entry ], [ %i123, %loop ]
%i30 = phi float [ 0.000000e+00, %entry ], [ %i124, %loop ]
%i31 = phi float [ 0.000000e+00, %entry ], [ %i125, %loop ]
%i32 = phi float [ 0.000000e+00, %entry ], [ %i132, %loop ]
%i33 = phi float [ 0.000000e+00, %entry ], [ %i133, %loop ]
%i34 = phi float [ 0.000000e+00, %entry ], [ %i134, %loop ]
%i35 = phi float [ 0.000000e+00, %entry ], [ %i135, %loop ]
%i36 = phi float [ 0.000000e+00, %entry ], [ %i140, %loop ]
%i37 = phi float [ 0.000000e+00, %entry ], [ %i141, %loop ]
%i38 = phi float [ 0.000000e+00, %entry ], [ %i146, %loop ]
%i39 = phi float [ 0.000000e+00, %entry ], [ %i147, %loop ]
%i40 = phi float [ 0.000000e+00, %entry ], [ %i152, %loop ]
%i41 = phi float [ 0.000000e+00, %entry ], [ %i40, %loop ]
%p0 = phi float [ 0.000000e+00, %entry ], [ %e0, %loop ]
%p1 = phi float [ 0.000000e+00, %entry ], [ %e1, %loop ]
%i42 = phi <2 x half> [ %i1, %entry ], [ zeroinitializer, %loop ]
%i43 = phi <2 x half> [ %i2, %entry ], [ zeroinitializer, %loop ]
%i44 = phi <8 x half> [ zeroinitializer, %entry ], [ <half 0.000000e+00, half 0.000000e+00, half poison, half poison, half poison, half poison, half poison, half poison>, %loop ]
%i45 = shufflevector <2 x half> %i42, <2 x half> %i43, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
%i46 = shufflevector <8 x half> %i45, <8 x half> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
%i47 = shufflevector <8 x half> %i45, <8 x half> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
%i48 = insertelement <4 x float> zeroinitializer, float %i3, i64 2
%i49 = insertelement <4 x float> %i48, float %i4, i64 3
%i50 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %i46, <8 x half> zeroinitializer, <4 x float> %i49, i32 0, i32 0, i32 0)
%i51 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i50, i32 0, i32 0, i32 0)
%i52 = extractelement <4 x float> %i51, i64 2
%i53 = extractelement <4 x float> %i51, i64 3
%i54 = insertelement <4 x float> zeroinitializer, float %i5, i64 0
%i55 = insertelement <4 x float> %i54, float %i6, i64 1
%i56 = insertelement <4 x float> %i55, float %i7, i64 2
%i57 = insertelement <4 x float> %i56, float %i8, i64 3
%i58 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i57, i32 0, i32 0, i32 0)
%i59 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i58, i32 0, i32 0, i32 0)
%i60 = extractelement <4 x float> %i59, i64 0
%i61 = extractelement <4 x float> %i59, i64 1
%i62 = extractelement <4 x float> %i59, i64 2
%i63 = extractelement <4 x float> %i59, i64 3
%i64 = insertelement <4 x float> zeroinitializer, float %i9, i64 0
%i65 = insertelement <4 x float> %i64, float %i10, i64 1
%i66 = insertelement <4 x float> %i65, float %i11, i64 2
%i67 = insertelement <4 x float> %i66, float %i12, i64 3
%i68 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i67, i32 0, i32 0, i32 0)
%i69 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i68, i32 0, i32 0, i32 0)
%i70 = extractelement <4 x float> %i69, i64 0
%i71 = extractelement <4 x float> %i69, i64 1
%i72 = extractelement <4 x float> %i69, i64 2
%i73 = extractelement <4 x float> %i69, i64 3
%i74 = insertelement <4 x float> zeroinitializer, float %i13, i64 0
%i75 = insertelement <4 x float> %i74, float %i14, i64 1
%i76 = insertelement <4 x float> %i75, float %i15, i64 2
%i77 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i76, i32 0, i32 0, i32 0)
%i78 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i77, i32 0, i32 0, i32 0)
%i79 = extractelement <4 x float> %i78, i64 0
%i80 = extractelement <4 x float> %i78, i64 1
%i81 = extractelement <4 x float> %i78, i64 2
%i82 = insertelement <4 x float> zeroinitializer, float %i16, i64 2
%i83 = insertelement <4 x float> %i82, float %i17, i64 3
%i84 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i83, i32 0, i32 0, i32 0)
%i85 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i84, i32 0, i32 0, i32 0)
%i86 = extractelement <4 x float> %i85, i64 2
%i87 = extractelement <4 x float> %i85, i64 3
%i88 = insertelement <4 x float> zeroinitializer, float %i18, i64 0
%i89 = insertelement <4 x float> %i88, float %i19, i64 1
%i90 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i89, i32 0, i32 0, i32 0)
%i91 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i90, i32 0, i32 0, i32 0)
%i92 = extractelement <4 x float> %i91, i64 0
%i93 = extractelement <4 x float> %i91, i64 1
%i94 = shufflevector <8 x half> zeroinitializer, <8 x half> %i44, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
%i95 = insertelement <4 x float> zeroinitializer, float %i20, i64 0
%i96 = insertelement <4 x float> %i95, float %i21, i64 1
%i97 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %i94, <8 x half> zeroinitializer, <4 x float> %i96, i32 0, i32 0, i32 0)
%i98 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i97, i32 0, i32 0, i32 0)
%i99 = extractelement <4 x float> %i98, i64 0
%i100 = extractelement <4 x float> %i98, i64 1
%i101 = insertelement <4 x float> poison, float %i22, i64 0
%i102 = insertelement <4 x float> %i101, float %i23, i64 0
%i103 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i102, i32 0, i32 0, i32 0)
%i104 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i103, i32 0, i32 0, i32 0)
%i105 = extractelement <4 x float> %i104, i64 0
%i106 = insertelement <4 x float> zeroinitializer, float %i24, i64 0
%i107 = insertelement <4 x float> %i106, float %i25, i64 1
%i108 = insertelement <4 x float> %i107, float %i26, i64 2
%i109 = insertelement <4 x float> %i108, float %i27, i64 3
%i110 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %i47, <8 x half> zeroinitializer, <4 x float> %i109, i32 0, i32 0, i32 0)
%i111 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i110, i32 0, i32 0, i32 0)
%i112 = extractelement <4 x float> %i111, i64 0
%i113 = extractelement <4 x float> %i111, i64 1
%i114 = extractelement <4 x float> %i111, i64 2
%i115 = extractelement <4 x float> %i111, i64 3
%i116 = insertelement <4 x float> zeroinitializer, float %i28, i64 0
%i117 = insertelement <4 x float> %i116, float %i29, i64 1
%i118 = insertelement <4 x float> %i117, float %i30, i64 2
%i119 = insertelement <4 x float> %i118, float %i31, i64 3
%i120 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i119, i32 0, i32 0, i32 0)
%i121 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i120, i32 0, i32 0, i32 0)
%i122 = extractelement <4 x float> %i121, i64 0
%i123 = extractelement <4 x float> %i121, i64 1
%i124 = extractelement <4 x float> %i121, i64 2
%i125 = extractelement <4 x float> %i121, i64 3
%i126 = insertelement <4 x float> zeroinitializer, float %i32, i64 0
%i127 = insertelement <4 x float> %i126, float %i33, i64 1
%i128 = insertelement <4 x float> %i127, float %i34, i64 2
%i129 = insertelement <4 x float> %i128, float %i35, i64 3
%i130 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i129, i32 0, i32 0, i32 0)
%i131 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i130, i32 0, i32 0, i32 0)
%i132 = extractelement <4 x float> %i131, i64 0
%i133 = extractelement <4 x float> %i131, i64 1
%i134 = extractelement <4 x float> %i131, i64 2
%i135 = extractelement <4 x float> %i131, i64 3
%i136 = insertelement <4 x float> zeroinitializer, float %i36, i64 0
%i137 = insertelement <4 x float> %i136, float %i37, i64 1
%i138 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i137, i32 0, i32 0, i32 0)
%i139 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i138, i32 0, i32 0, i32 0)
%i140 = extractelement <4 x float> %i139, i64 0
%i141 = extractelement <4 x float> %i139, i64 1
%i142 = insertelement <4 x float> zeroinitializer, float %i38, i64 0
%i143 = insertelement <4 x float> %i142, float %i39, i64 1
%i144 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i143, i32 0, i32 0, i32 0)
%i145 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i144, i32 0, i32 0, i32 0)
%i146 = extractelement <4 x float> %i145, i64 0
%i147 = extractelement <4 x float> %i145, i64 1
%i148 = insertelement <4 x float> <float 0.000000e+00, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00>, float %i40, i64 2
%i149 = insertelement <4 x float> %i148, float %i41, i64 3
%i150 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i149, i32 0, i32 0, i32 0)
%i151 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %i150, i32 0, i32 0, i32 0)
%i152 = extractelement <4 x float> %i151, i64 0
%x0 = insertelement <4 x float> zeroinitializer, float %p0, i64 0
%x1 = insertelement <4 x float> %x0, float %p1, i64 1
%x2 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %x1, i32 0, i32 0, i32 0)
%x3 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> zeroinitializer, <8 x half> zeroinitializer, <4 x float> %x2, i32 0, i32 0, i32 0)
%e0 = extractelement <4 x float> %x3, i64 0
%e1 = extractelement <4 x float> %x3, i64 1
br label %loop
}
declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half>, <8 x half>, <4 x float>, i32 immarg, i32 immarg, i32 immarg)
attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }