Google Git
Sign in
llvm / llvm-project / 46b4ca8e9869ad6e51b9d22bb08e32b37ccff03a / . / llvm / test / CodeGen / AMDGPU / llvm.amdgcn.sched.group.barrier.iterative.ll
blob: 371b4f070094d68fbf9ffcc1d581058917583ad9 [file] [log] [blame]
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs -misched-cluster=0 -misched=gcn-iterative-minreg < %s | FileCheck -check-prefix=GCN-MINREG %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs -misched-cluster=0 -misched=gcn-iterative-max-occupancy-experimental < %s | FileCheck -check-prefix=GCN-MAXOCC %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs -misched-cluster=0 -misched=gcn-iterative-ilp < %s | FileCheck -check-prefix=GCN-ILP %s
define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave(ptr addrspace(3) noalias %in, ptr addrspace(3) noalias %out) #0 {
; GCN-MINREG-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave:
; GCN-MINREG: ; %bb.0: ; %entry
; GCN-MINREG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-MINREG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-MINREG-NEXT: v_lshlrev_b32_e32 v0, 7, v0
; GCN-MINREG-NEXT: v_mov_b32_e32 v2, 1.0
; GCN-MINREG-NEXT: v_mov_b32_e32 v1, 2.0
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_add_u32_e32 v4, s0, v0
; GCN-MINREG-NEXT: ds_read_b128 a[28:31], v4 offset:112
; GCN-MINREG-NEXT: ds_read_b128 a[24:27], v4 offset:96
; GCN-MINREG-NEXT: ds_read_b128 a[20:23], v4 offset:80
; GCN-MINREG-NEXT: ds_read_b128 a[16:19], v4 offset:64
; GCN-MINREG-NEXT: ds_read_b128 a[0:3], v4
; GCN-MINREG-NEXT: ds_read_b128 a[4:7], v4 offset:16
; GCN-MINREG-NEXT: ds_read_b128 a[8:11], v4 offset:32
; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v4 offset:48
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; GCN-MINREG-NEXT: v_add_u32_e32 v5, s1, v0
; GCN-MINREG-NEXT: v_mov_b32_e32 v0, s1
; GCN-MINREG-NEXT: v_add_u32_e32 v3, 0x6000, v4
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: ds_write_b128 v5, a[28:31] offset:112
; GCN-MINREG-NEXT: ds_write_b128 v5, a[24:27] offset:96
; GCN-MINREG-NEXT: ds_write_b128 v5, a[20:23] offset:80
; GCN-MINREG-NEXT: ds_write_b128 v5, a[16:19] offset:64
; GCN-MINREG-NEXT: ds_write_b128 v5, a[12:15] offset:48
; GCN-MINREG-NEXT: ds_write_b128 v5, a[8:11] offset:32
; GCN-MINREG-NEXT: ds_write_b128 v5, a[4:7] offset:16
; GCN-MINREG-NEXT: ds_write_b128 v5, a[0:3]
; GCN-MINREG-NEXT: ds_read_b128 a[28:31], v4 offset:8304
; GCN-MINREG-NEXT: ds_read_b128 a[24:27], v4 offset:8288
; GCN-MINREG-NEXT: ds_read_b128 a[20:23], v4 offset:8272
; GCN-MINREG-NEXT: ds_read_b128 a[16:19], v4 offset:8256
; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v4 offset:8240
; GCN-MINREG-NEXT: ds_read_b128 a[8:11], v4 offset:8224
; GCN-MINREG-NEXT: ds_read_b128 a[4:7], v4 offset:8208
; GCN-MINREG-NEXT: ds_read_b128 a[0:3], v4 offset:8192
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 2
; GCN-MINREG-NEXT: ds_write_b128 v0, a[24:27] offset:8288
; GCN-MINREG-NEXT: ds_write_b128 v0, a[28:31] offset:8304
; GCN-MINREG-NEXT: ds_write_b128 v0, a[16:19] offset:8256
; GCN-MINREG-NEXT: ds_write_b128 v0, a[20:23] offset:8272
; GCN-MINREG-NEXT: ds_write_b128 v0, a[8:11] offset:8224
; GCN-MINREG-NEXT: ds_write_b128 v0, a[12:15] offset:8240
; GCN-MINREG-NEXT: ds_write_b128 v0, a[0:3] offset:8192
; GCN-MINREG-NEXT: ds_write_b128 v0, a[4:7] offset:8208
; GCN-MINREG-NEXT: ds_read_b128 a[28:31], v4 offset:24688
; GCN-MINREG-NEXT: ds_read_b128 a[24:27], v4 offset:24672
; GCN-MINREG-NEXT: ds_read_b128 a[20:23], v4 offset:24656
; GCN-MINREG-NEXT: ds_read_b128 a[16:19], v4 offset:24640
; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v4 offset:24624
; GCN-MINREG-NEXT: ds_read_b128 a[8:11], v4 offset:24608
; GCN-MINREG-NEXT: ds_read_b128 a[4:7], v4 offset:24592
; GCN-MINREG-NEXT: ds_read_b128 a[0:3], v4 offset:24576
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 2
; GCN-MINREG-NEXT: ds_write_b128 v0, a[24:27] offset:16480
; GCN-MINREG-NEXT: ds_write_b128 v0, a[28:31] offset:16496
; GCN-MINREG-NEXT: ds_write_b128 v0, a[16:19] offset:16448
; GCN-MINREG-NEXT: ds_write_b128 v0, a[20:23] offset:16464
; GCN-MINREG-NEXT: ds_write_b128 v0, a[8:11] offset:16416
; GCN-MINREG-NEXT: ds_write_b128 v0, a[12:15] offset:16432
; GCN-MINREG-NEXT: ds_write_b128 v0, a[0:3] offset:16384
; GCN-MINREG-NEXT: ds_write_b128 v0, a[4:7] offset:16400
; GCN-MINREG-NEXT: ds_read_b128 a[28:31], v4 offset:49264
; GCN-MINREG-NEXT: ds_read_b128 a[24:27], v4 offset:49248
; GCN-MINREG-NEXT: ds_read_b128 a[20:23], v4 offset:49232
; GCN-MINREG-NEXT: ds_read_b128 a[16:19], v4 offset:49216
; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v4 offset:49200
; GCN-MINREG-NEXT: ds_read_b128 a[8:11], v4 offset:49184
; GCN-MINREG-NEXT: ds_read_b128 a[4:7], v4 offset:49168
; GCN-MINREG-NEXT: ds_read_b128 a[0:3], v4 offset:49152
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 2
; GCN-MINREG-NEXT: ds_write_b128 v0, a[24:27] offset:24672
; GCN-MINREG-NEXT: ds_write_b128 v0, a[28:31] offset:24688
; GCN-MINREG-NEXT: ds_write_b128 v0, a[16:19] offset:24640
; GCN-MINREG-NEXT: ds_write_b128 v0, a[20:23] offset:24656
; GCN-MINREG-NEXT: ds_write_b128 v0, a[8:11] offset:24608
; GCN-MINREG-NEXT: ds_write_b128 v0, a[12:15] offset:24624
; GCN-MINREG-NEXT: ds_write_b128 v0, a[0:3] offset:24576
; GCN-MINREG-NEXT: ds_write_b128 v0, a[4:7] offset:24592
; GCN-MINREG-NEXT: ds_read_b128 a[28:31], v3 offset:57456
; GCN-MINREG-NEXT: ds_read_b128 a[24:27], v3 offset:57440
; GCN-MINREG-NEXT: ds_read_b128 a[20:23], v3 offset:57424
; GCN-MINREG-NEXT: ds_read_b128 a[16:19], v3 offset:57408
; GCN-MINREG-NEXT: ds_read_b128 a[0:3], v3 offset:57344
; GCN-MINREG-NEXT: ds_read_b128 a[4:7], v3 offset:57360
; GCN-MINREG-NEXT: ds_read_b128 a[8:11], v3 offset:57376
; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v3 offset:57392
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31]
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 2
; GCN-MINREG-NEXT: ds_write_b128 v0, a[24:27] offset:32864
; GCN-MINREG-NEXT: ds_write_b128 v0, a[28:31] offset:32880
; GCN-MINREG-NEXT: ds_write_b128 v0, a[16:19] offset:32832
; GCN-MINREG-NEXT: ds_write_b128 v0, a[20:23] offset:32848
; GCN-MINREG-NEXT: ds_write_b128 v0, a[8:11] offset:32800
; GCN-MINREG-NEXT: ds_write_b128 v0, a[12:15] offset:32816
; GCN-MINREG-NEXT: ds_write_b128 v0, a[0:3] offset:32768
; GCN-MINREG-NEXT: ds_write_b128 v0, a[4:7] offset:32784
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MINREG-NEXT: s_endpgm
;
; GCN-MAXOCC-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave:
; GCN-MAXOCC: ; %bb.0: ; %entry
; GCN-MAXOCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-MAXOCC-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-MAXOCC-NEXT: v_lshlrev_b32_e32 v1, 7, v0
; GCN-MAXOCC-NEXT: v_mov_b32_e32 v2, 1.0
; GCN-MAXOCC-NEXT: v_mov_b32_e32 v3, 2.0
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_add_u32_e32 v0, s0, v1
; GCN-MAXOCC-NEXT: ds_read_b128 a[28:31], v0 offset:112
; GCN-MAXOCC-NEXT: ds_read_b128 a[24:27], v0 offset:96
; GCN-MAXOCC-NEXT: ds_read_b128 a[20:23], v0 offset:80
; GCN-MAXOCC-NEXT: ds_read_b128 a[16:19], v0 offset:64
; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0
; GCN-MAXOCC-NEXT: ds_read_b128 a[4:7], v0 offset:16
; GCN-MAXOCC-NEXT: ds_read_b128 a[8:11], v0 offset:32
; GCN-MAXOCC-NEXT: ds_read_b128 a[12:15], v0 offset:48
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; GCN-MAXOCC-NEXT: v_add_u32_e32 v1, s1, v1
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 1
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[28:31] offset:112
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[24:27] offset:96
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[20:23] offset:80
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[16:19] offset:64
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[12:15] offset:48
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[8:11] offset:32
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[4:7] offset:16
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[0:3]
; GCN-MAXOCC-NEXT: ds_read_b128 a[28:31], v0 offset:8304
; GCN-MAXOCC-NEXT: ds_read_b128 a[24:27], v0 offset:8288
; GCN-MAXOCC-NEXT: ds_read_b128 a[20:23], v0 offset:8272
; GCN-MAXOCC-NEXT: ds_read_b128 a[16:19], v0 offset:8256
; GCN-MAXOCC-NEXT: ds_read_b128 a[12:15], v0 offset:8240
; GCN-MAXOCC-NEXT: ds_read_b128 a[8:11], v0 offset:8224
; GCN-MAXOCC-NEXT: ds_read_b128 a[4:7], v0 offset:8208
; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0 offset:8192
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; GCN-MAXOCC-NEXT: v_mov_b32_e32 v1, s1
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 1
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[24:27] offset:8288
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[28:31] offset:8304
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[16:19] offset:8256
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[20:23] offset:8272
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[8:11] offset:8224
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[12:15] offset:8240
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[0:3] offset:8192
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[4:7] offset:8208
; GCN-MAXOCC-NEXT: ds_read_b128 a[28:31], v0 offset:24688
; GCN-MAXOCC-NEXT: ds_read_b128 a[24:27], v0 offset:24672
; GCN-MAXOCC-NEXT: ds_read_b128 a[20:23], v0 offset:24656
; GCN-MAXOCC-NEXT: ds_read_b128 a[16:19], v0 offset:24640
; GCN-MAXOCC-NEXT: ds_read_b128 a[12:15], v0 offset:24624
; GCN-MAXOCC-NEXT: ds_read_b128 a[8:11], v0 offset:24608
; GCN-MAXOCC-NEXT: ds_read_b128 a[4:7], v0 offset:24592
; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0 offset:24576
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 2
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[24:27] offset:16480
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[28:31] offset:16496
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[16:19] offset:16448
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[20:23] offset:16464
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[8:11] offset:16416
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[12:15] offset:16432
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[0:3] offset:16384
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[4:7] offset:16400
; GCN-MAXOCC-NEXT: ds_read_b128 a[28:31], v0 offset:49264
; GCN-MAXOCC-NEXT: ds_read_b128 a[24:27], v0 offset:49248
; GCN-MAXOCC-NEXT: ds_read_b128 a[20:23], v0 offset:49232
; GCN-MAXOCC-NEXT: ds_read_b128 a[16:19], v0 offset:49216
; GCN-MAXOCC-NEXT: ds_read_b128 a[12:15], v0 offset:49200
; GCN-MAXOCC-NEXT: ds_read_b128 a[8:11], v0 offset:49184
; GCN-MAXOCC-NEXT: ds_read_b128 a[4:7], v0 offset:49168
; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0 offset:49152
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; GCN-MAXOCC-NEXT: v_add_u32_e32 v0, 0x6000, v0
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 1
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[24:27] offset:24672
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[28:31] offset:24688
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[16:19] offset:24640
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[20:23] offset:24656
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[8:11] offset:24608
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[12:15] offset:24624
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[0:3] offset:24576
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[4:7] offset:24592
; GCN-MAXOCC-NEXT: ds_read_b128 a[28:31], v0 offset:57456
; GCN-MAXOCC-NEXT: ds_read_b128 a[24:27], v0 offset:57440
; GCN-MAXOCC-NEXT: ds_read_b128 a[20:23], v0 offset:57424
; GCN-MAXOCC-NEXT: ds_read_b128 a[16:19], v0 offset:57408
; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0 offset:57344
; GCN-MAXOCC-NEXT: ds_read_b128 a[4:7], v0 offset:57360
; GCN-MAXOCC-NEXT: ds_read_b128 a[8:11], v0 offset:57376
; GCN-MAXOCC-NEXT: ds_read_b128 a[12:15], v0 offset:57392
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31]
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 2
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[24:27] offset:32864
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[28:31] offset:32880
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[16:19] offset:32832
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[20:23] offset:32848
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[8:11] offset:32800
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[12:15] offset:32816
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[0:3] offset:32768
; GCN-MAXOCC-NEXT: ds_write_b128 v1, a[4:7] offset:32784
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: s_endpgm
;
; GCN-ILP-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave:
; GCN-ILP: ; %bb.0: ; %entry
; GCN-ILP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-ILP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-ILP-NEXT: v_lshlrev_b32_e32 v0, 7, v0
; GCN-ILP-NEXT: v_mov_b32_e32 v1, 1.0
; GCN-ILP-NEXT: v_mov_b32_e32 v2, 2.0
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_add_u32_e32 v3, s0, v0
; GCN-ILP-NEXT: ds_read_b128 a[12:15], v3 offset:48
; GCN-ILP-NEXT: ds_read_b128 a[8:11], v3 offset:32
; GCN-ILP-NEXT: ds_read_b128 a[4:7], v3 offset:16
; GCN-ILP-NEXT: ds_read_b128 a[0:3], v3
; GCN-ILP-NEXT: ds_read_b128 a[16:19], v3 offset:64
; GCN-ILP-NEXT: ds_read_b128 a[20:23], v3 offset:80
; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:96
; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:112
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
; GCN-ILP-NEXT: v_add_u32_e32 v0, s1, v0
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 1
; GCN-ILP-NEXT: ds_write_b128 v0, a[28:31] offset:112
; GCN-ILP-NEXT: ds_write_b128 v0, a[24:27] offset:96
; GCN-ILP-NEXT: ds_write_b128 v0, a[20:23] offset:80
; GCN-ILP-NEXT: ds_write_b128 v0, a[16:19] offset:64
; GCN-ILP-NEXT: ds_write_b128 v0, a[12:15] offset:48
; GCN-ILP-NEXT: ds_write_b128 v0, a[8:11] offset:32
; GCN-ILP-NEXT: ds_write_b128 v0, a[4:7] offset:16
; GCN-ILP-NEXT: ds_write_b128 v0, a[0:3]
; GCN-ILP-NEXT: ds_read_b128 a[0:3], v3 offset:8192
; GCN-ILP-NEXT: ds_read_b128 a[4:7], v3 offset:8208
; GCN-ILP-NEXT: ds_read_b128 a[8:11], v3 offset:8224
; GCN-ILP-NEXT: ds_read_b128 a[12:15], v3 offset:8240
; GCN-ILP-NEXT: ds_read_b128 a[16:19], v3 offset:8256
; GCN-ILP-NEXT: ds_read_b128 a[20:23], v3 offset:8272
; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:8288
; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:8304
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
; GCN-ILP-NEXT: v_mov_b32_e32 v0, s1
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 1
; GCN-ILP-NEXT: ds_write_b128 v0, a[24:27] offset:8288
; GCN-ILP-NEXT: ds_write_b128 v0, a[28:31] offset:8304
; GCN-ILP-NEXT: ds_write_b128 v0, a[16:19] offset:8256
; GCN-ILP-NEXT: ds_write_b128 v0, a[20:23] offset:8272
; GCN-ILP-NEXT: ds_write_b128 v0, a[8:11] offset:8224
; GCN-ILP-NEXT: ds_write_b128 v0, a[12:15] offset:8240
; GCN-ILP-NEXT: ds_write_b128 v0, a[0:3] offset:8192
; GCN-ILP-NEXT: ds_write_b128 v0, a[4:7] offset:8208
; GCN-ILP-NEXT: ds_read_b128 a[0:3], v3 offset:24576
; GCN-ILP-NEXT: ds_read_b128 a[4:7], v3 offset:24592
; GCN-ILP-NEXT: ds_read_b128 a[8:11], v3 offset:24608
; GCN-ILP-NEXT: ds_read_b128 a[12:15], v3 offset:24624
; GCN-ILP-NEXT: ds_read_b128 a[16:19], v3 offset:24640
; GCN-ILP-NEXT: ds_read_b128 a[20:23], v3 offset:24656
; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:24672
; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:24688
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 2
; GCN-ILP-NEXT: ds_write_b128 v0, a[4:7] offset:16400
; GCN-ILP-NEXT: ds_read_b128 a[4:7], v3 offset:49168
; GCN-ILP-NEXT: ds_write_b128 v0, a[0:3] offset:16384
; GCN-ILP-NEXT: ds_read_b128 a[0:3], v3 offset:49152
; GCN-ILP-NEXT: ds_write_b128 v0, a[12:15] offset:16432
; GCN-ILP-NEXT: ds_read_b128 a[12:15], v3 offset:49200
; GCN-ILP-NEXT: ds_write_b128 v0, a[8:11] offset:16416
; GCN-ILP-NEXT: ds_read_b128 a[8:11], v3 offset:49184
; GCN-ILP-NEXT: ds_write_b128 v0, a[20:23] offset:16464
; GCN-ILP-NEXT: ds_read_b128 a[20:23], v3 offset:49232
; GCN-ILP-NEXT: ds_write_b128 v0, a[16:19] offset:16448
; GCN-ILP-NEXT: ds_read_b128 a[16:19], v3 offset:49216
; GCN-ILP-NEXT: ds_write_b128 v0, a[28:31] offset:16496
; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:49264
; GCN-ILP-NEXT: ds_write_b128 v0, a[24:27] offset:16480
; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:49248
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
; GCN-ILP-NEXT: v_add_u32_e32 v3, 0x6000, v3
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 1
; GCN-ILP-NEXT: ds_write_b128 v0, a[4:7] offset:24592
; GCN-ILP-NEXT: ds_read_b128 a[4:7], v3 offset:57360
; GCN-ILP-NEXT: ds_write_b128 v0, a[0:3] offset:24576
; GCN-ILP-NEXT: ds_read_b128 a[0:3], v3 offset:57344
; GCN-ILP-NEXT: ds_write_b128 v0, a[12:15] offset:24624
; GCN-ILP-NEXT: ds_read_b128 a[12:15], v3 offset:57392
; GCN-ILP-NEXT: ds_write_b128 v0, a[8:11] offset:24608
; GCN-ILP-NEXT: ds_read_b128 a[8:11], v3 offset:57376
; GCN-ILP-NEXT: ds_write_b128 v0, a[20:23] offset:24656
; GCN-ILP-NEXT: ds_read_b128 a[20:23], v3 offset:57424
; GCN-ILP-NEXT: ds_write_b128 v0, a[16:19] offset:24640
; GCN-ILP-NEXT: ds_read_b128 a[16:19], v3 offset:57408
; GCN-ILP-NEXT: ds_write_b128 v0, a[28:31] offset:24688
; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:57456
; GCN-ILP-NEXT: ds_write_b128 v0, a[24:27] offset:24672
; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:57440
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 2
; GCN-ILP-NEXT: ds_write_b128 v0, a[24:27] offset:32864
; GCN-ILP-NEXT: ds_write_b128 v0, a[28:31] offset:32880
; GCN-ILP-NEXT: ds_write_b128 v0, a[16:19] offset:32832
; GCN-ILP-NEXT: ds_write_b128 v0, a[20:23] offset:32848
; GCN-ILP-NEXT: ds_write_b128 v0, a[8:11] offset:32800
; GCN-ILP-NEXT: ds_write_b128 v0, a[12:15] offset:32816
; GCN-ILP-NEXT: ds_write_b128 v0, a[0:3] offset:32768
; GCN-ILP-NEXT: ds_write_b128 v0, a[4:7] offset:32784
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-ILP-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
%load.0.addr = getelementptr <32 x float>, ptr addrspace(3) %in, i32 %idx
%load.0 = load <32 x float>, ptr addrspace(3) %load.0.addr
%load.1.addr = getelementptr <32 x float>, ptr addrspace(3) %load.0.addr, i32 64
%load.1 = load <32 x float>, ptr addrspace(3) %load.1.addr
%load.2.addr = getelementptr <32 x float>, ptr addrspace(3) %load.1.addr, i32 128
%load.2 = load <32 x float>, ptr addrspace(3) %load.2.addr
%load.3.addr = getelementptr <32 x float>, ptr addrspace(3) %load.2.addr, i32 192
%load.3 = load <32 x float>, ptr addrspace(3) %load.3.addr
%load.4.addr = getelementptr <32 x float>, ptr addrspace(3) %load.3.addr, i32 256
%load.4 = load <32 x float>, ptr addrspace(3) %load.4.addr
%mai.0 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %load.0, i32 0, i32 0, i32 0)
%mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %load.1, i32 0, i32 0, i32 0)
%mai.2 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %load.2, i32 0, i32 0, i32 0)
%mai.3 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %load.3, i32 0, i32 0, i32 0)
%mai.4 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %load.4, i32 0, i32 0, i32 0)
%store.0.addr = getelementptr <32 x float>, ptr addrspace(3) %out, i32 %idx
store <32 x float> %mai.0, ptr addrspace(3) %store.0.addr
%store.1.addr = getelementptr <32 x float>, ptr addrspace(3) %out, i32 64
store <32 x float> %mai.1, ptr addrspace(3) %store.1.addr
%store.2.addr = getelementptr <32 x float>, ptr addrspace(3) %out, i32 128
store <32 x float> %mai.2, ptr addrspace(3) %store.2.addr
%store.3.addr = getelementptr <32 x float>, ptr addrspace(3) %out, i32 192
store <32 x float> %mai.3, ptr addrspace(3) %store.3.addr
%store.4.addr = getelementptr <32 x float>, ptr addrspace(3) %out, i32 256
store <32 x float> %mai.4, ptr addrspace(3) %store.4.addr
; 8 DS read
call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 8, i32 0)
; 1 MFMA
call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)
; 8 DS write
call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 8, i32 0)
; 8 DS read
call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 8, i32 0)
; 1 MFMA
call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)
; 8 DS write
call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 8, i32 0)
; 8 DS read
call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 8, i32 0)
; 1 MFMA
call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)
; 8 DS write
call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 8, i32 0)
; 8 DS read
call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 8, i32 0)
; 1 MFMA
call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)
; 8 DS write
call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 8, i32 0)
; 8 DS read
call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 8, i32 0)
; 1 MFMA
call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)
; 8 DS write
call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 8, i32 0)
ret void
}
define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_interleave_split_region(ptr addrspace(3) noalias %in, ptr addrspace(3) noalias %out) #0 {
; GCN-MINREG-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave_split_region:
; GCN-MINREG: ; %bb.0: ; %entry
; GCN-MINREG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-MINREG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-MINREG-NEXT: v_lshlrev_b32_e32 v2, 7, v0
; GCN-MINREG-NEXT: v_mov_b32_e32 v1, 1.0
; GCN-MINREG-NEXT: v_mov_b32_e32 v0, 2.0
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_add_u32_e32 v3, s0, v2
; GCN-MINREG-NEXT: ds_read_b128 a[28:31], v3 offset:112
; GCN-MINREG-NEXT: ds_read_b128 a[24:27], v3 offset:96
; GCN-MINREG-NEXT: ds_read_b128 a[20:23], v3 offset:80
; GCN-MINREG-NEXT: ds_read_b128 a[16:19], v3 offset:64
; GCN-MINREG-NEXT: ds_read_b128 a[0:3], v3
; GCN-MINREG-NEXT: ds_read_b128 a[4:7], v3 offset:16
; GCN-MINREG-NEXT: ds_read_b128 a[8:11], v3 offset:32
; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v3 offset:48
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v0, a[0:31]
; GCN-MINREG-NEXT: v_add_u32_e32 v2, s1, v2
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 1
; GCN-MINREG-NEXT: ds_write_b128 v2, a[28:31] offset:112
; GCN-MINREG-NEXT: ds_write_b128 v2, a[24:27] offset:96
; GCN-MINREG-NEXT: ds_write_b128 v2, a[20:23] offset:80
; GCN-MINREG-NEXT: ds_write_b128 v2, a[16:19] offset:64
; GCN-MINREG-NEXT: ds_write_b128 v2, a[12:15] offset:48
; GCN-MINREG-NEXT: ds_write_b128 v2, a[8:11] offset:32
; GCN-MINREG-NEXT: ds_write_b128 v2, a[4:7] offset:16
; GCN-MINREG-NEXT: ds_write_b128 v2, a[0:3]
; GCN-MINREG-NEXT: ds_read_b128 a[28:31], v3 offset:8304
; GCN-MINREG-NEXT: ds_read_b128 a[24:27], v3 offset:8288
; GCN-MINREG-NEXT: ds_read_b128 a[20:23], v3 offset:8272
; GCN-MINREG-NEXT: ds_read_b128 a[16:19], v3 offset:8256
; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v3 offset:8240
; GCN-MINREG-NEXT: ds_read_b128 a[8:11], v3 offset:8224
; GCN-MINREG-NEXT: ds_read_b128 a[4:7], v3 offset:8208
; GCN-MINREG-NEXT: ds_read_b128 a[0:3], v3 offset:8192
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v0, a[0:31]
; GCN-MINREG-NEXT: v_mov_b32_e32 v2, s1
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 1
; GCN-MINREG-NEXT: ds_write_b128 v2, a[24:27] offset:8288
; GCN-MINREG-NEXT: ds_write_b128 v2, a[28:31] offset:8304
; GCN-MINREG-NEXT: ds_write_b128 v2, a[16:19] offset:8256
; GCN-MINREG-NEXT: ds_write_b128 v2, a[20:23] offset:8272
; GCN-MINREG-NEXT: ds_write_b128 v2, a[8:11] offset:8224
; GCN-MINREG-NEXT: ds_write_b128 v2, a[12:15] offset:8240
; GCN-MINREG-NEXT: ds_write_b128 v2, a[0:3] offset:8192
; GCN-MINREG-NEXT: ds_write_b128 v2, a[4:7] offset:8208
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_barrier mask(0x00000000)
; GCN-MINREG-NEXT: ds_read_b128 a[28:31], v3 offset:24688
; GCN-MINREG-NEXT: ds_read_b128 a[24:27], v3 offset:24672
; GCN-MINREG-NEXT: ds_read_b128 a[20:23], v3 offset:24656
; GCN-MINREG-NEXT: ds_read_b128 a[16:19], v3 offset:24640
; GCN-MINREG-NEXT: ds_read_b128 a[0:3], v3 offset:24576
; GCN-MINREG-NEXT: ds_read_b128 a[4:7], v3 offset:24592
; GCN-MINREG-NEXT: ds_read_b128 a[8:11], v3 offset:24608
; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v3 offset:24624
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v0, a[0:31]
; GCN-MINREG-NEXT: v_add_u32_e32 v4, 0x6000, v3
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 1
; GCN-MINREG-NEXT: ds_write_b128 v2, a[28:31] offset:16496
; GCN-MINREG-NEXT: ds_write_b128 v2, a[24:27] offset:16480
; GCN-MINREG-NEXT: ds_write_b128 v2, a[20:23] offset:16464
; GCN-MINREG-NEXT: ds_write_b128 v2, a[16:19] offset:16448
; GCN-MINREG-NEXT: ds_write_b128 v2, a[12:15] offset:16432
; GCN-MINREG-NEXT: ds_write_b128 v2, a[8:11] offset:16416
; GCN-MINREG-NEXT: ds_write_b128 v2, a[4:7] offset:16400
; GCN-MINREG-NEXT: ds_write_b128 v2, a[0:3] offset:16384
; GCN-MINREG-NEXT: ds_read_b128 a[28:31], v3 offset:49264
; GCN-MINREG-NEXT: ds_read_b128 a[24:27], v3 offset:49248
; GCN-MINREG-NEXT: ds_read_b128 a[20:23], v3 offset:49232
; GCN-MINREG-NEXT: ds_read_b128 a[16:19], v3 offset:49216
; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v3 offset:49200
; GCN-MINREG-NEXT: ds_read_b128 a[8:11], v3 offset:49184
; GCN-MINREG-NEXT: ds_read_b128 a[4:7], v3 offset:49168
; GCN-MINREG-NEXT: ds_read_b128 a[0:3], v3 offset:49152
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v0, a[0:31]
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 2
; GCN-MINREG-NEXT: ds_write_b128 v2, a[28:31] offset:24688
; GCN-MINREG-NEXT: ds_write_b128 v2, a[24:27] offset:24672
; GCN-MINREG-NEXT: ds_write_b128 v2, a[20:23] offset:24656
; GCN-MINREG-NEXT: ds_write_b128 v2, a[16:19] offset:24640
; GCN-MINREG-NEXT: ds_write_b128 v2, a[12:15] offset:24624
; GCN-MINREG-NEXT: ds_write_b128 v2, a[8:11] offset:24608
; GCN-MINREG-NEXT: ds_write_b128 v2, a[4:7] offset:24592
; GCN-MINREG-NEXT: ds_write_b128 v2, a[0:3] offset:24576
; GCN-MINREG-NEXT: ds_read_b128 a[28:31], v4 offset:57456
; GCN-MINREG-NEXT: ds_read_b128 a[24:27], v4 offset:57440
; GCN-MINREG-NEXT: ds_read_b128 a[20:23], v4 offset:57424
; GCN-MINREG-NEXT: ds_read_b128 a[16:19], v4 offset:57408
; GCN-MINREG-NEXT: ds_read_b128 a[0:3], v4 offset:57344
; GCN-MINREG-NEXT: ds_read_b128 a[4:7], v4 offset:57360
; GCN-MINREG-NEXT: ds_read_b128 a[8:11], v4 offset:57376
; GCN-MINREG-NEXT: ds_read_b128 a[12:15], v4 offset:57392
; GCN-MINREG-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MINREG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v0, a[0:31]
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 7
; GCN-MINREG-NEXT: s_nop 2
; GCN-MINREG-NEXT: ds_write_b128 v2, a[28:31] offset:32880
; GCN-MINREG-NEXT: ds_write_b128 v2, a[24:27] offset:32864
; GCN-MINREG-NEXT: ds_write_b128 v2, a[20:23] offset:32848
; GCN-MINREG-NEXT: ds_write_b128 v2, a[16:19] offset:32832
; GCN-MINREG-NEXT: ds_write_b128 v2, a[12:15] offset:32816
; GCN-MINREG-NEXT: ds_write_b128 v2, a[8:11] offset:32800
; GCN-MINREG-NEXT: ds_write_b128 v2, a[4:7] offset:32784
; GCN-MINREG-NEXT: ds_write_b128 v2, a[0:3] offset:32768
; GCN-MINREG-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MINREG-NEXT: s_endpgm
;
; GCN-MAXOCC-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave_split_region:
; GCN-MAXOCC: ; %bb.0: ; %entry
; GCN-MAXOCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-MAXOCC-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-MAXOCC-NEXT: v_lshlrev_b32_e32 v3, 7, v0
; GCN-MAXOCC-NEXT: v_mov_b32_e32 v1, 1.0
; GCN-MAXOCC-NEXT: v_mov_b32_e32 v2, 2.0
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_add_u32_e32 v0, s0, v3
; GCN-MAXOCC-NEXT: ds_read_b128 a[28:31], v0 offset:112
; GCN-MAXOCC-NEXT: ds_read_b128 a[24:27], v0 offset:96
; GCN-MAXOCC-NEXT: ds_read_b128 a[20:23], v0 offset:80
; GCN-MAXOCC-NEXT: ds_read_b128 a[16:19], v0 offset:64
; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0
; GCN-MAXOCC-NEXT: ds_read_b128 a[4:7], v0 offset:16
; GCN-MAXOCC-NEXT: ds_read_b128 a[8:11], v0 offset:32
; GCN-MAXOCC-NEXT: ds_read_b128 a[12:15], v0 offset:48
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
; GCN-MAXOCC-NEXT: v_add_u32_e32 v3, s1, v3
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 1
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[28:31] offset:112
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[24:27] offset:96
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[20:23] offset:80
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[16:19] offset:64
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[12:15] offset:48
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[8:11] offset:32
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[4:7] offset:16
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[0:3]
; GCN-MAXOCC-NEXT: ds_read_b128 a[28:31], v0 offset:8304
; GCN-MAXOCC-NEXT: ds_read_b128 a[24:27], v0 offset:8288
; GCN-MAXOCC-NEXT: ds_read_b128 a[20:23], v0 offset:8272
; GCN-MAXOCC-NEXT: ds_read_b128 a[16:19], v0 offset:8256
; GCN-MAXOCC-NEXT: ds_read_b128 a[12:15], v0 offset:8240
; GCN-MAXOCC-NEXT: ds_read_b128 a[8:11], v0 offset:8224
; GCN-MAXOCC-NEXT: ds_read_b128 a[4:7], v0 offset:8208
; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0 offset:8192
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
; GCN-MAXOCC-NEXT: v_mov_b32_e32 v3, s1
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 1
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[24:27] offset:8288
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[28:31] offset:8304
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[16:19] offset:8256
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[20:23] offset:8272
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[8:11] offset:8224
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[12:15] offset:8240
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[0:3] offset:8192
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[4:7] offset:8208
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_barrier mask(0x00000000)
; GCN-MAXOCC-NEXT: ds_read_b128 a[28:31], v0 offset:24688
; GCN-MAXOCC-NEXT: ds_read_b128 a[24:27], v0 offset:24672
; GCN-MAXOCC-NEXT: ds_read_b128 a[20:23], v0 offset:24656
; GCN-MAXOCC-NEXT: ds_read_b128 a[16:19], v0 offset:24640
; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0 offset:24576
; GCN-MAXOCC-NEXT: ds_read_b128 a[4:7], v0 offset:24592
; GCN-MAXOCC-NEXT: ds_read_b128 a[8:11], v0 offset:24608
; GCN-MAXOCC-NEXT: ds_read_b128 a[12:15], v0 offset:24624
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 2
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[28:31] offset:16496
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[24:27] offset:16480
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[20:23] offset:16464
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[16:19] offset:16448
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[12:15] offset:16432
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[8:11] offset:16416
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[4:7] offset:16400
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[0:3] offset:16384
; GCN-MAXOCC-NEXT: ds_read_b128 a[28:31], v0 offset:49264
; GCN-MAXOCC-NEXT: ds_read_b128 a[24:27], v0 offset:49248
; GCN-MAXOCC-NEXT: ds_read_b128 a[20:23], v0 offset:49232
; GCN-MAXOCC-NEXT: ds_read_b128 a[16:19], v0 offset:49216
; GCN-MAXOCC-NEXT: ds_read_b128 a[12:15], v0 offset:49200
; GCN-MAXOCC-NEXT: ds_read_b128 a[8:11], v0 offset:49184
; GCN-MAXOCC-NEXT: ds_read_b128 a[4:7], v0 offset:49168
; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0 offset:49152
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
; GCN-MAXOCC-NEXT: v_add_u32_e32 v0, 0x6000, v0
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 1
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[28:31] offset:24688
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[24:27] offset:24672
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[20:23] offset:24656
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[16:19] offset:24640
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[12:15] offset:24624
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[8:11] offset:24608
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[4:7] offset:24592
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[0:3] offset:24576
; GCN-MAXOCC-NEXT: ds_read_b128 a[28:31], v0 offset:57456
; GCN-MAXOCC-NEXT: ds_read_b128 a[24:27], v0 offset:57440
; GCN-MAXOCC-NEXT: ds_read_b128 a[20:23], v0 offset:57424
; GCN-MAXOCC-NEXT: ds_read_b128 a[16:19], v0 offset:57408
; GCN-MAXOCC-NEXT: ds_read_b128 a[0:3], v0 offset:57344
; GCN-MAXOCC-NEXT: ds_read_b128 a[4:7], v0 offset:57360
; GCN-MAXOCC-NEXT: ds_read_b128 a[8:11], v0 offset:57376
; GCN-MAXOCC-NEXT: ds_read_b128 a[12:15], v0 offset:57392
; GCN-MAXOCC-NEXT: s_waitcnt lgkmcnt(0)
; GCN-MAXOCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v2, a[0:31]
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 7
; GCN-MAXOCC-NEXT: s_nop 2
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[28:31] offset:32880
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[24:27] offset:32864
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[20:23] offset:32848
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[16:19] offset:32832
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[12:15] offset:32816
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[8:11] offset:32800
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[4:7] offset:32784
; GCN-MAXOCC-NEXT: ds_write_b128 v3, a[0:3] offset:32768
; GCN-MAXOCC-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-MAXOCC-NEXT: s_endpgm
;
; GCN-ILP-LABEL: test_sched_group_barrier_pipeline_MFMA_interleave_split_region:
; GCN-ILP: ; %bb.0: ; %entry
; GCN-ILP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-ILP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-ILP-NEXT: v_lshlrev_b32_e32 v2, 7, v0
; GCN-ILP-NEXT: v_mov_b32_e32 v0, 1.0
; GCN-ILP-NEXT: v_mov_b32_e32 v1, 2.0
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_add_u32_e32 v3, s0, v2
; GCN-ILP-NEXT: ds_read_b128 a[12:15], v3 offset:48
; GCN-ILP-NEXT: ds_read_b128 a[8:11], v3 offset:32
; GCN-ILP-NEXT: ds_read_b128 a[4:7], v3 offset:16
; GCN-ILP-NEXT: ds_read_b128 a[0:3], v3
; GCN-ILP-NEXT: ds_read_b128 a[16:19], v3 offset:64
; GCN-ILP-NEXT: ds_read_b128 a[20:23], v3 offset:80
; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:96
; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:112
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; GCN-ILP-NEXT: v_add_u32_e32 v2, s1, v2
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 1
; GCN-ILP-NEXT: ds_write_b128 v2, a[0:3]
; GCN-ILP-NEXT: ds_read_b128 a[0:3], v3 offset:8192
; GCN-ILP-NEXT: ds_write_b128 v2, a[4:7] offset:16
; GCN-ILP-NEXT: ds_read_b128 a[4:7], v3 offset:8208
; GCN-ILP-NEXT: ds_write_b128 v2, a[8:11] offset:32
; GCN-ILP-NEXT: ds_read_b128 a[8:11], v3 offset:8224
; GCN-ILP-NEXT: ds_write_b128 v2, a[12:15] offset:48
; GCN-ILP-NEXT: ds_read_b128 a[12:15], v3 offset:8240
; GCN-ILP-NEXT: ds_write_b128 v2, a[16:19] offset:64
; GCN-ILP-NEXT: ds_read_b128 a[16:19], v3 offset:8256
; GCN-ILP-NEXT: ds_write_b128 v2, a[20:23] offset:80
; GCN-ILP-NEXT: ds_read_b128 a[20:23], v3 offset:8272
; GCN-ILP-NEXT: ds_write_b128 v2, a[24:27] offset:96
; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:8288
; GCN-ILP-NEXT: ds_write_b128 v2, a[28:31] offset:112
; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:8304
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; GCN-ILP-NEXT: v_mov_b32_e32 v2, s1
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 1
; GCN-ILP-NEXT: ds_write_b128 v2, a[24:27] offset:8288
; GCN-ILP-NEXT: ds_write_b128 v2, a[28:31] offset:8304
; GCN-ILP-NEXT: ds_write_b128 v2, a[16:19] offset:8256
; GCN-ILP-NEXT: ds_write_b128 v2, a[20:23] offset:8272
; GCN-ILP-NEXT: ds_write_b128 v2, a[8:11] offset:8224
; GCN-ILP-NEXT: ds_write_b128 v2, a[12:15] offset:8240
; GCN-ILP-NEXT: ds_write_b128 v2, a[0:3] offset:8192
; GCN-ILP-NEXT: ds_write_b128 v2, a[4:7] offset:8208
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_barrier mask(0x00000000)
; GCN-ILP-NEXT: ds_read_b128 a[12:15], v3 offset:24624
; GCN-ILP-NEXT: ds_read_b128 a[8:11], v3 offset:24608
; GCN-ILP-NEXT: ds_read_b128 a[4:7], v3 offset:24592
; GCN-ILP-NEXT: ds_read_b128 a[0:3], v3 offset:24576
; GCN-ILP-NEXT: ds_read_b128 a[16:19], v3 offset:24640
; GCN-ILP-NEXT: ds_read_b128 a[20:23], v3 offset:24656
; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:24672
; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:24688
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 2
; GCN-ILP-NEXT: ds_write_b128 v2, a[28:31] offset:16496
; GCN-ILP-NEXT: ds_write_b128 v2, a[24:27] offset:16480
; GCN-ILP-NEXT: ds_write_b128 v2, a[20:23] offset:16464
; GCN-ILP-NEXT: ds_write_b128 v2, a[16:19] offset:16448
; GCN-ILP-NEXT: ds_write_b128 v2, a[12:15] offset:16432
; GCN-ILP-NEXT: ds_write_b128 v2, a[8:11] offset:16416
; GCN-ILP-NEXT: ds_write_b128 v2, a[4:7] offset:16400
; GCN-ILP-NEXT: ds_write_b128 v2, a[0:3] offset:16384
; GCN-ILP-NEXT: ds_read_b128 a[0:3], v3 offset:49152
; GCN-ILP-NEXT: ds_read_b128 a[4:7], v3 offset:49168
; GCN-ILP-NEXT: ds_read_b128 a[8:11], v3 offset:49184
; GCN-ILP-NEXT: ds_read_b128 a[12:15], v3 offset:49200
; GCN-ILP-NEXT: ds_read_b128 a[16:19], v3 offset:49216
; GCN-ILP-NEXT: ds_read_b128 a[20:23], v3 offset:49232
; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:49248
; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:49264
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; GCN-ILP-NEXT: v_add_u32_e32 v3, 0x6000, v3
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 1
; GCN-ILP-NEXT: ds_write_b128 v2, a[0:3] offset:24576
; GCN-ILP-NEXT: ds_read_b128 a[0:3], v3 offset:57344
; GCN-ILP-NEXT: ds_write_b128 v2, a[4:7] offset:24592
; GCN-ILP-NEXT: ds_read_b128 a[4:7], v3 offset:57360
; GCN-ILP-NEXT: ds_write_b128 v2, a[8:11] offset:24608
; GCN-ILP-NEXT: ds_read_b128 a[8:11], v3 offset:57376
; GCN-ILP-NEXT: ds_write_b128 v2, a[12:15] offset:24624
; GCN-ILP-NEXT: ds_read_b128 a[12:15], v3 offset:57392
; GCN-ILP-NEXT: ds_write_b128 v2, a[16:19] offset:24640
; GCN-ILP-NEXT: ds_read_b128 a[16:19], v3 offset:57408
; GCN-ILP-NEXT: ds_write_b128 v2, a[20:23] offset:24656
; GCN-ILP-NEXT: ds_read_b128 a[20:23], v3 offset:57424
; GCN-ILP-NEXT: ds_write_b128 v2, a[24:27] offset:24672
; GCN-ILP-NEXT: ds_read_b128 a[24:27], v3 offset:57440
; GCN-ILP-NEXT: ds_write_b128 v2, a[28:31] offset:24688
; GCN-ILP-NEXT: ds_read_b128 a[28:31], v3 offset:57456
; GCN-ILP-NEXT: s_waitcnt lgkmcnt(0)
; GCN-ILP-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000100) size(8) SyncID(0)
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 7
; GCN-ILP-NEXT: s_nop 2
; GCN-ILP-NEXT: ds_write_b128 v2, a[28:31] offset:32880
; GCN-ILP-NEXT: ds_write_b128 v2, a[24:27] offset:32864
; GCN-ILP-NEXT: ds_write_b128 v2, a[20:23] offset:32848
; GCN-ILP-NEXT: ds_write_b128 v2, a[16:19] offset:32832
; GCN-ILP-NEXT: ds_write_b128 v2, a[12:15] offset:32816
; GCN-ILP-NEXT: ds_write_b128 v2, a[8:11] offset:32800
; GCN-ILP-NEXT: ds_write_b128 v2, a[4:7] offset:32784
; GCN-ILP-NEXT: ds_write_b128 v2, a[0:3] offset:32768
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)
; GCN-ILP-NEXT: ; sched_group_barrier mask(0x00000200) size(8) SyncID(0)
; GCN-ILP-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
%load.0.addr = getelementptr <32 x float>, ptr addrspace(3) %in, i32 %idx
%load.0 = load <32 x float>, ptr addrspace(3) %load.0.addr
%load.1.addr = getelementptr <32 x float>, ptr addrspace(3) %load.0.addr, i32 64
%load.1 = load <32 x float>, ptr addrspace(3) %load.1.addr
%mai.0 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %load.0, i32 0, i32 0, i32 0)
%mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %load.1, i32 0, i32 0, i32 0)
%store.0.addr = getelementptr <32 x float>, ptr addrspace(3) %out, i32 %idx
store <32 x float> %mai.0, ptr addrspace(3) %store.0.addr
%store.1.addr = getelementptr <32 x float>, ptr addrspace(3) %out, i32 64
store <32 x float> %mai.1, ptr addrspace(3) %store.1.addr
; 8 DS read
call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 8, i32 0)
; 1 MFMA
call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)
; 8 DS write
call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 8, i32 0)
; 8 DS read
call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 8, i32 0)
; 1 MFMA
call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)
; 8 DS write
call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 8, i32 0)
; parition the regions
call void @llvm.amdgcn.sched.barrier(i32 0)
%load.2.addr = getelementptr <32 x float>, ptr addrspace(3) %load.1.addr, i32 128
%load.2 = load <32 x float>, ptr addrspace(3) %load.2.addr
%load.3.addr = getelementptr <32 x float>, ptr addrspace(3) %load.2.addr, i32 192
%load.3 = load <32 x float>, ptr addrspace(3) %load.3.addr
%load.4.addr = getelementptr <32 x float>, ptr addrspace(3) %load.3.addr, i32 256
%load.4 = load <32 x float>, ptr addrspace(3) %load.4.addr
%mai.2 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %load.2, i32 0, i32 0, i32 0)
%mai.3 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %load.3, i32 0, i32 0, i32 0)
%mai.4 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %load.4, i32 0, i32 0, i32 0)
%store.2.addr = getelementptr <32 x float>, ptr addrspace(3) %out, i32 128
store <32 x float> %mai.2, ptr addrspace(3) %store.2.addr
%store.3.addr = getelementptr <32 x float>, ptr addrspace(3) %out, i32 192
store <32 x float> %mai.3, ptr addrspace(3) %store.3.addr
%store.4.addr = getelementptr <32 x float>, ptr addrspace(3) %out, i32 256
store <32 x float> %mai.4, ptr addrspace(3) %store.4.addr
; 8 DS read
call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 8, i32 0)
; 1 MFMA
call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)
; 8 DS write
call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 8, i32 0)
; 8 DS read
call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 8, i32 0)
; 1 MFMA
call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)
; 8 DS write
call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 8, i32 0)
; 8 DS read
call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 8, i32 0)
; 1 MFMA
call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)
; 8 DS write
call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 8, i32 0)
ret void
}