llvm/test/CodeGen/AMDGPU/coexec-block-carried-latency.mir

# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=amdgpu12.50 -amdgpu-sched-strategy=coexec --run-pass=machine-scheduler -verify-misched %s -o - | FileCheck -check-prefix=DEFAULT %s
# RUN: llc -mtriple=amdgpu12.50 -amdgpu-sched-strategy=coexec -amdgpu-block-carried-latency=fence --run-pass=machine-scheduler -verify-misched %s -o - | FileCheck -check-prefix=FENCE %s
# RUN: llc -mtriple=amdgpu12.50 -amdgpu-sched-strategy=coexec -amdgpu-block-carried-latency=all --run-pass=machine-scheduler -verify-misched %s -o - | FileCheck -check-prefix=ALL %s

--- |
  define void @test-block-carried-latency() #0 { ret void }

  attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="1,128" }
...

---
name: test-block-carried-latency
tracksRegLiveness: true
body: |
  ; DEFAULT-LABEL: name: test-block-carried-latency
  ; DEFAULT: bb.0:
  ; DEFAULT-NEXT:   successors: %bb.1(0x80000000)
  ; DEFAULT-NEXT:   liveins: $vgpr0, $sgpr0, $sgpr1
  ; DEFAULT-NEXT: {{  $}}
  ; DEFAULT-NEXT:   [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 0, 0, implicit $exec
  ; DEFAULT-NEXT:   [[DS_READ_B32_gfx9_1:%[0-9]+]]:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 4, 0, implicit $exec
  ; DEFAULT-NEXT:   [[DS_READ_B32_gfx9_2:%[0-9]+]]:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 8, 0, implicit $exec
  ; DEFAULT-NEXT:   [[DS_READ_B32_gfx9_3:%[0-9]+]]:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 12, 0, implicit $exec
  ; DEFAULT-NEXT:   [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
  ; DEFAULT-NEXT:   [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
  ; DEFAULT-NEXT:   [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
  ; DEFAULT-NEXT:   [[DEF3:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
  ; DEFAULT-NEXT:   [[DEF4:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
  ; DEFAULT-NEXT:   [[DEF5:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
  ; DEFAULT-NEXT:   S_BRANCH %bb.1
  ; DEFAULT-NEXT: {{  $}}
  ; DEFAULT-NEXT: bb.1:
  ; DEFAULT-NEXT:   liveins: $vgpr0, $sgpr0, $sgpr1
  ; DEFAULT-NEXT: {{  $}}
  ; DEFAULT-NEXT:   ATOMIC_FENCE 5, 2
  ; DEFAULT-NEXT:   early-clobber %10:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DS_READ_B32_gfx9_]], [[DS_READ_B32_gfx9_1]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; DEFAULT-NEXT:   dead [[S_ADD_I32_:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; DEFAULT-NEXT:   dead [[S_ADD_I32_1:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; DEFAULT-NEXT:   dead [[S_ADD_I32_2:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; DEFAULT-NEXT:   dead [[S_ADD_I32_3:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; DEFAULT-NEXT:   dead [[S_ADD_I32_4:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; DEFAULT-NEXT:   dead [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; DEFAULT-NEXT:   dead [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; DEFAULT-NEXT:   early-clobber %11:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; DEFAULT-NEXT:   dead [[S_ADD_I32_5:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; DEFAULT-NEXT:   dead [[S_ADD_I32_6:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; DEFAULT-NEXT:   dead [[V_ADD_U32_e64_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; DEFAULT-NEXT:   dead [[S_ADD_I32_7:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; DEFAULT-NEXT:   dead [[S_ADD_I32_8:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; DEFAULT-NEXT:   dead [[V_ADD_U32_e64_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; DEFAULT-NEXT:   dead [[V_ADD_U32_e64_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; DEFAULT-NEXT:   dead early-clobber %28:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; DEFAULT-NEXT:   dead [[S_ADD_I32_9:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; DEFAULT-NEXT:   dead [[V_ADD_U32_e64_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; DEFAULT-NEXT:   dead early-clobber %29:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; DEFAULT-NEXT:   dead early-clobber %30:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; DEFAULT-NEXT:   dead early-clobber %31:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; DEFAULT-NEXT:   dead early-clobber %32:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; DEFAULT-NEXT:   S_ENDPGM 0, implicit %10, implicit %11
  ;
  ; FENCE-LABEL: name: test-block-carried-latency
  ; FENCE: bb.0:
  ; FENCE-NEXT:   successors: %bb.1(0x80000000)
  ; FENCE-NEXT:   liveins: $vgpr0, $sgpr0, $sgpr1
  ; FENCE-NEXT: {{  $}}
  ; FENCE-NEXT:   [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 0, 0, implicit $exec
  ; FENCE-NEXT:   [[DS_READ_B32_gfx9_1:%[0-9]+]]:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 4, 0, implicit $exec
  ; FENCE-NEXT:   [[DS_READ_B32_gfx9_2:%[0-9]+]]:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 8, 0, implicit $exec
  ; FENCE-NEXT:   [[DS_READ_B32_gfx9_3:%[0-9]+]]:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 12, 0, implicit $exec
  ; FENCE-NEXT:   [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
  ; FENCE-NEXT:   [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
  ; FENCE-NEXT:   [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
  ; FENCE-NEXT:   [[DEF3:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
  ; FENCE-NEXT:   [[DEF4:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
  ; FENCE-NEXT:   [[DEF5:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
  ; FENCE-NEXT:   S_BRANCH %bb.1
  ; FENCE-NEXT: {{  $}}
  ; FENCE-NEXT: bb.1:
  ; FENCE-NEXT:   liveins: $vgpr0, $sgpr0, $sgpr1
  ; FENCE-NEXT: {{  $}}
  ; FENCE-NEXT:   early-clobber %10:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DS_READ_B32_gfx9_]], [[DS_READ_B32_gfx9_1]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; FENCE-NEXT:   dead [[S_ADD_I32_:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; FENCE-NEXT:   dead [[S_ADD_I32_1:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; FENCE-NEXT:   dead [[S_ADD_I32_2:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; FENCE-NEXT:   dead [[S_ADD_I32_3:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; FENCE-NEXT:   dead [[S_ADD_I32_4:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; FENCE-NEXT:   dead [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; FENCE-NEXT:   dead [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; FENCE-NEXT:   early-clobber %11:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; FENCE-NEXT:   dead [[S_ADD_I32_5:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; FENCE-NEXT:   dead [[S_ADD_I32_6:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; FENCE-NEXT:   dead [[V_ADD_U32_e64_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; FENCE-NEXT:   dead [[S_ADD_I32_7:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; FENCE-NEXT:   dead [[S_ADD_I32_8:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; FENCE-NEXT:   dead [[V_ADD_U32_e64_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; FENCE-NEXT:   dead [[V_ADD_U32_e64_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; FENCE-NEXT:   dead early-clobber %28:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; FENCE-NEXT:   dead [[S_ADD_I32_9:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; FENCE-NEXT:   dead [[V_ADD_U32_e64_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; FENCE-NEXT:   ATOMIC_FENCE 5, 2
  ; FENCE-NEXT:   dead early-clobber %29:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; FENCE-NEXT:   dead early-clobber %30:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; FENCE-NEXT:   dead early-clobber %31:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; FENCE-NEXT:   dead early-clobber %32:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; FENCE-NEXT:   S_ENDPGM 0, implicit %10, implicit %11
  ;
  ; ALL-LABEL: name: test-block-carried-latency
  ; ALL: bb.0:
  ; ALL-NEXT:   successors: %bb.1(0x80000000)
  ; ALL-NEXT:   liveins: $vgpr0, $sgpr0, $sgpr1
  ; ALL-NEXT: {{  $}}
  ; ALL-NEXT:   [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 0, 0, implicit $exec
  ; ALL-NEXT:   [[DS_READ_B32_gfx9_1:%[0-9]+]]:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 4, 0, implicit $exec
  ; ALL-NEXT:   [[DS_READ_B32_gfx9_2:%[0-9]+]]:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 8, 0, implicit $exec
  ; ALL-NEXT:   [[DS_READ_B32_gfx9_3:%[0-9]+]]:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 12, 0, implicit $exec
  ; ALL-NEXT:   [[DEF:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
  ; ALL-NEXT:   [[DEF1:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
  ; ALL-NEXT:   [[DEF2:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
  ; ALL-NEXT:   [[DEF3:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
  ; ALL-NEXT:   [[DEF4:%[0-9]+]]:vreg_512_align2 = IMPLICIT_DEF
  ; ALL-NEXT:   [[DEF5:%[0-9]+]]:vreg_256_align2 = IMPLICIT_DEF
  ; ALL-NEXT:   S_BRANCH %bb.1
  ; ALL-NEXT: {{  $}}
  ; ALL-NEXT: bb.1:
  ; ALL-NEXT:   liveins: $vgpr0, $sgpr0, $sgpr1
  ; ALL-NEXT: {{  $}}
  ; ALL-NEXT:   dead [[S_ADD_I32_:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; ALL-NEXT:   dead [[S_ADD_I32_1:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; ALL-NEXT:   dead [[S_ADD_I32_2:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; ALL-NEXT:   dead [[S_ADD_I32_3:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; ALL-NEXT:   dead [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; ALL-NEXT:   dead [[S_ADD_I32_4:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; ALL-NEXT:   dead [[V_ADD_U32_e64_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; ALL-NEXT:   dead [[S_ADD_I32_5:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; ALL-NEXT:   dead [[V_ADD_U32_e64_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; ALL-NEXT:   dead [[S_ADD_I32_6:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; ALL-NEXT:   dead [[V_ADD_U32_e64_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; ALL-NEXT:   dead [[S_ADD_I32_7:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; ALL-NEXT:   dead [[V_ADD_U32_e64_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; ALL-NEXT:   dead [[S_ADD_I32_8:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; ALL-NEXT:   dead [[V_ADD_U32_e64_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
  ; ALL-NEXT:   dead [[S_ADD_I32_9:%[0-9]+]]:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
  ; ALL-NEXT:   early-clobber %10:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF]], [[DEF1]], 0, [[DEF2]], [[DS_READ_B32_gfx9_]], [[DS_READ_B32_gfx9_1]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; ALL-NEXT:   ATOMIC_FENCE 5, 2
  ; ALL-NEXT:   early-clobber %11:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; ALL-NEXT:   dead early-clobber %28:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; ALL-NEXT:   dead early-clobber %29:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; ALL-NEXT:   dead early-clobber %30:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; ALL-NEXT:   dead early-clobber %31:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; ALL-NEXT:   dead early-clobber %32:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr [[DEF3]], [[DEF4]], 0, [[DEF5]], [[DS_READ_B32_gfx9_2]], [[DS_READ_B32_gfx9_3]], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
  ; ALL-NEXT:   S_ENDPGM 0, implicit %10, implicit %11
  bb.0:
    successors: %bb.1
    liveins: $vgpr0, $sgpr0, $sgpr1

    %0:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 0, 0, implicit $exec
    %1:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 4, 0, implicit $exec
    %2:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 8, 0, implicit $exec
    %3:vgpr_32_lo256 = DS_READ_B32_gfx9 $vgpr0, 12, 0, implicit $exec

    %10:vreg_512_align2 = IMPLICIT_DEF
    %11:vreg_512_align2 = IMPLICIT_DEF
    %12:vreg_256_align2 = IMPLICIT_DEF
    %15:vreg_512_align2 = IMPLICIT_DEF
    %16:vreg_512_align2 = IMPLICIT_DEF
    %17:vreg_256_align2 = IMPLICIT_DEF

    S_BRANCH %bb.1

  bb.1:
    liveins: $vgpr0, $sgpr0, $sgpr1

    ATOMIC_FENCE 5, 2

    %20:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %10, %11, 0, %12, %0, %1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
    %21:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %15, %16, 0, %17, %2, %3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec

    %22:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
    %23:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
    %24:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
    %25:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
    %26:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
    %27:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
    %28:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
    %29:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
    %30:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
    %31:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
    %32:vgpr_32 = V_ADD_U32_e64 $vgpr0, $vgpr0, 0, implicit $exec
    %33:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
    %34:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
    %35:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
    %36:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
    %37:sgpr_32 = S_ADD_I32 $sgpr0, $sgpr1, implicit-def dead $scc
    %38:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %15, %16, 0, %17, %2, %3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
    %39:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %15, %16, 0, %17, %2, %3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
    %40:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %15, %16, 0, %17, %2, %3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
    %41:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %15, %16, 0, %17, %2, %3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
    %42:vreg_256_align2 = V_WMMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_w32_threeaddr %15, %16, 0, %17, %2, %3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
    S_ENDPGM 0, implicit %20, implicit %21
...