Google Git
Sign in
llvm/llvm-project/ba767d0bbbde4107700ff66ecfd97eb75d85a35d/./llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
blob: 4e5ad79b72468928d0cff79d9e0e45741b4d38b6 [file]
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s -check-prefixes=SDAG
; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s -check-prefixes=GISEL
; Test async mark/wait with global_load_lds and global loads
; This version uses wave barriers to enforce program order so that unrelated vmem
; instructions do not get reordered before reaching this point.
define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out) {
; SDAG-LABEL: interleaved_with_wave_barrier:
; SDAG: ; %bb.0: ; %entry
; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v9, v4
; SDAG-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v6, v5
; SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 0x54, v[0:1]
; SDAG-NEXT: v_add_nc_u32_e32 v3, 0x54, v2
; SDAG-NEXT: global_load_b32 v10, v[8:9], off offset:44
; SDAG-NEXT: global_load_b32 v11, v[0:1], off offset:4
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_NT nv
; SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 0x58, v[8:9]
; SDAG-NEXT: v_add_nc_u32_e32 v3, 0x58, v2
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:8
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_LU nv
; SDAG-NEXT: ; wave barrier
; SDAG-NEXT: global_load_b32 v1, v[8:9], off offset:48
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(1)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: ds_load_b32 v3, v2 offset:84
; SDAG-NEXT: ; wait_asyncmark(0)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v2, v2 offset:88
; SDAG-NEXT: s_wait_loadcnt 0x2
; SDAG-NEXT: v_add_nc_u32_e32 v4, v11, v10
; SDAG-NEXT: s_wait_loadcnt_dscnt 0x101
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; SDAG-NEXT: v_add3_u32 v0, v4, v3, v0
; SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; SDAG-NEXT: v_add3_u32 v0, v0, v1, v2
; SDAG-NEXT: global_store_b32 v[6:7], v0, off
; SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GISEL-LABEL: interleaved_with_wave_barrier:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GISEL-NEXT: s_wait_kmcnt 0x0
; GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
; GISEL-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
; GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0x54, v0
; GISEL-NEXT: v_add_nc_u32_e32 v3, 0x54, v2
; GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
; GISEL-NEXT: global_load_b32 v10, v[8:9], off offset:44
; GISEL-NEXT: global_load_b32 v11, v[0:1], off offset:4
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_NT nv
; GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0x58, v8
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v9, vcc_lo
; GISEL-NEXT: v_add_nc_u32_e32 v3, 0x58, v2
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:8
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_LU nv
; GISEL-NEXT: ; wave barrier
; GISEL-NEXT: global_load_b32 v1, v[8:9], off offset:48
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(1)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: ds_load_b32 v3, v2 offset:84
; GISEL-NEXT: ; wait_asyncmark(0)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v2, v2 offset:88
; GISEL-NEXT: s_wait_loadcnt 0x2
; GISEL-NEXT: v_add_nc_u32_e32 v6, v11, v10
; GISEL-NEXT: s_wait_loadcnt_dscnt 0x101
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GISEL-NEXT: v_add3_u32 v0, v6, v3, v0
; GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GISEL-NEXT: v_add3_u32 v0, v0, v1, v2
; GISEL-NEXT: global_store_b32 v[4:5], v0, off
; GISEL-NEXT: s_set_pc_i64 s[30:31]
entry:
; First batch: global load, global load, async global-to-LDS
%bar_gep11 = getelementptr i32, ptr addrspace(1) %bar, i32 11
%bar_v11 = load i32, ptr addrspace(1) %bar_gep11
%foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
%foo_v1 = load i32, ptr addrspace(1) %foo_gep1
%lds_gep21 = getelementptr i32, ptr addrspace(3) %lds, i32 21
%bar_gep21 = getelementptr i32, ptr addrspace(1) %foo, i32 21
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar_gep21, ptr addrspace(3) %lds_gep21, i32 4, i32 u0x21)
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.asyncmark()
; Second batch: global load, async global-to-LDS, global load
%foo_gep2 = getelementptr i32, ptr addrspace(1) %foo, i32 2
%foo_v2 = load i32, ptr addrspace(1) %foo_gep2
%bar_gep22 = getelementptr i32, ptr addrspace(1) %bar, i32 22
%lds_gep22 = getelementptr i32, ptr addrspace(3) %lds, i32 22
call void @llvm.amdgcn.wave.barrier()
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar_gep22, ptr addrspace(3) %lds_gep22, i32 4, i32 u0x23)
call void @llvm.amdgcn.wave.barrier()
%bar_gep12 = getelementptr i32, ptr addrspace(1) %bar, i32 12
%bar_v12 = load i32, ptr addrspace(1) %bar_gep12
call void @llvm.amdgcn.asyncmark()
; Wait for first async mark and read from LDS
call void @llvm.amdgcn.wait.asyncmark(i16 1)
%lds_val21 = load i32, ptr addrspace(3) %lds_gep21
; Wait for the next async mark.
; Notable that the asyncmark is sufficient to prevent the optimizer from coalescing the previous ds_load with the next one.
call void @llvm.amdgcn.wait.asyncmark(i16 0)
%lds_val22 = load i32, ptr addrspace(3) %lds_gep22
%sum1 = add i32 %foo_v1, %bar_v11
%sum2 = add i32 %sum1, %lds_val21
%sum3 = add i32 %sum2, %foo_v2
; Finally a loadcnt(0) for %bar_v12, which was not included in the async mark that followed it.
%sum4 = add i32 %sum3, %bar_v12
%sum5 = add i32 %sum4, %lds_val22
store i32 %sum5, ptr addrspace(1) %out
ret void
}
; A perfect loop that is unlikely to exist in real life. It uses only async
; operations, and result in waits that exactly match the stream of
; those outstanding operations.
define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) {
; SDAG-LABEL: test_pipelined_loop:
; SDAG: ; %bb.0: ; %prolog
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; SDAG-NEXT: s_load_b32 s3, s[4:5], 0x44 nv
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; SDAG-NEXT: s_add_co_i32 s6, s2, 4
; SDAG-NEXT: s_mov_b32 s7, s2
; SDAG-NEXT: v_mov_b32_e32 v2, s6
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
; SDAG-NEXT: v_mov_b32_e32 v1, 4
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v1, s[0:1] offset:4 nv
; SDAG-NEXT: v_mov_b32_e32 v1, 0
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: .LBB1_1: ; %loop_body
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT: s_add_co_i32 s8, s7, 8
; SDAG-NEXT: s_add_co_i32 s6, s6, 1
; SDAG-NEXT: v_mov_b32_e32 v2, s8
; SDAG-NEXT: global_load_async_to_lds_b32 v2, v0, s[0:1] offset:4 nv
; SDAG-NEXT: v_mov_b32_e32 v2, s7
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(2)
; SDAG-NEXT: s_wait_asynccnt 0x2
; SDAG-NEXT: s_add_co_i32 s7, s7, 4
; SDAG-NEXT: s_cmp_lt_i32 s6, s3
; SDAG-NEXT: ds_load_b32 v2, v2
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4
; SDAG-NEXT: s_wait_dscnt 0x0
; SDAG-NEXT: v_add_nc_u32_e32 v1, v1, v2
; SDAG-NEXT: s_cbranch_scc1 .LBB1_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_lshl2_add_u32 s0, s3, s2
; SDAG-NEXT: ; wait_asyncmark(1)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: s_add_co_i32 s0, s0, -8
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s0
; SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
; SDAG-NEXT: ds_load_b32 v0, v0
; SDAG-NEXT: ; wait_asyncmark(0)
; SDAG-NEXT: s_wait_dscnt 0x0
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: v_add_nc_u32_e32 v0, v1, v0
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_pipelined_loop:
; GISEL: ; %bb.0: ; %prolog
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_clause 0x1
; GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GISEL-NEXT: s_load_b32 s3, s[4:5], 0x44 nv
; GISEL-NEXT: s_mov_b32 s7, 2
; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v2, 4
; GISEL-NEXT: s_wait_kmcnt 0x0
; GISEL-NEXT: v_mov_b32_e32 v1, s2
; GISEL-NEXT: s_add_co_u32 s6, s2, 4
; GISEL-NEXT: s_mov_b32 s8, s2
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
; GISEL-NEXT: v_mov_b32_e32 v1, s6
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: s_mov_b32 s6, 0
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v2, s[0:1] offset:4 nv
; GISEL-NEXT: s_add_co_u32 s0, s0, 8
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: .LBB1_1: ; %loop_body
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-NEXT: s_add_co_u32 s9, s8, 8
; GISEL-NEXT: s_add_co_i32 s7, s7, 1
; GISEL-NEXT: v_mov_b32_e32 v1, s9
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
; GISEL-NEXT: v_mov_b32_e32 v1, s8
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(2)
; GISEL-NEXT: s_wait_asynccnt 0x2
; GISEL-NEXT: ds_load_b32 v1, v1
; GISEL-NEXT: s_wait_dscnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s9, v1
; GISEL-NEXT: s_add_co_i32 s6, s6, s9
; GISEL-NEXT: s_add_co_u32 s0, s0, 4
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
; GISEL-NEXT: s_add_co_u32 s8, s8, 4
; GISEL-NEXT: s_cmp_lt_i32 s7, s3
; GISEL-NEXT: s_cbranch_scc1 .LBB1_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_lshl_b32 s0, s3, 2
; GISEL-NEXT: ; wait_asyncmark(1)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: s_add_co_u32 s0, s2, s0
; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_add_co_u32 s0, s0, -8
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_mov_b32_e32 v0, s0
; GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
; GISEL-NEXT: ds_load_b32 v0, v0
; GISEL-NEXT: ; wait_asyncmark(0)
; GISEL-NEXT: s_wait_dscnt 0x0
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s2, v0
; GISEL-NEXT: s_add_co_i32 s2, s6, s2
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_mov_b32_e32 v0, s2
; GISEL-NEXT: s_wait_kmcnt 0x0
; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
prolog:
; Load first iteration
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20)
call void @llvm.amdgcn.asyncmark()
; Load second iteration
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
%foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep1, ptr addrspace(3) %lds_gep1, i32 4, i32 u0x20)
call void @llvm.amdgcn.asyncmark()
br label %loop_body
loop_body:
%i = phi i32 [ 2, %prolog ], [ %i.next, %loop_body ]
%sum = phi i32 [ 0, %prolog ], [ %sum_i, %loop_body ]
; Load next iteration
%lds_gep_cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i
%foo_gep_cur = getelementptr i32, ptr addrspace(1) %foo, i32 %i
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep_cur, ptr addrspace(3) %lds_gep_cur, i32 4, i32 u0x20)
call void @llvm.amdgcn.asyncmark()
; Wait for iteration i-2 and process
call void @llvm.amdgcn.wait.asyncmark(i16 2)
%lds_idx = sub i32 %i, 2
%lds_gep_read = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_idx
%lds_val = load i32, ptr addrspace(3) %lds_gep_read
%sum_i = add i32 %sum, %lds_val
%i.next = add i32 %i, 1
%cmp = icmp slt i32 %i.next, %n
br i1 %cmp, label %loop_body, label %epilog
epilog:
; Process remaining iterations
call void @llvm.amdgcn.wait.asyncmark(i16 1)
%lds_n_2 = sub i32 %n, 2
%lds_gep_n_2 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_2
%lds_val_n_2 = load i32, ptr addrspace(3) %lds_gep_n_2
%sum_e2 = add i32 %sum_i, %lds_val_n_2
%out_gep_e1 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_2
call void @llvm.amdgcn.wait.asyncmark(i16 0)
%lds_n_1 = sub i32 %n, 1
%lds_gep_n_1 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_1
%lds_val_n_1 = load i32, ptr addrspace(3) %lds_gep_n_1
%sum_e1 = add i32 %sum_e2, %lds_val_n_1
store i32 %sum_e2, ptr addrspace(1) %bar
ret void
}
; Software pipelined loop with async global-to-LDS and global loads
define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) {
; SDAG-LABEL: test_pipelined_loop_with_global:
; SDAG: ; %bb.0: ; %prolog
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: s_load_b96 s[8:10], s[4:5], 0x24 nv
; SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
; SDAG-NEXT: v_mov_b32_e32 v0, 0
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: s_load_b32 s6, s[8:9], 0x0
; SDAG-NEXT: s_load_b32 s7, s[0:1], 0x0
; SDAG-NEXT: v_mov_b32_e32 v1, s10
; SDAG-NEXT: s_add_co_i32 s11, s10, 4
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; SDAG-NEXT: v_dual_mov_b32 v3, 4 :: v_dual_mov_b32 v4, s11
; SDAG-NEXT: s_load_b32 s11, s[4:5], 0x44 nv
; SDAG-NEXT: s_clause 0x2
; SDAG-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
; SDAG-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
; SDAG-NEXT: s_wait_xcnt 0x0
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 8
; SDAG-NEXT: s_add_nc_u64 s[4:5], s[8:9], 8
; SDAG-NEXT: s_wait_kmcnt 0x0
; SDAG-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v6, s7
; SDAG-NEXT: s_mov_b64 s[6:7], s[2:3]
; SDAG-NEXT: global_load_async_to_lds_b32 v4, v3, s[8:9] offset:4 nv
; SDAG-NEXT: s_wait_loadcnt 0x0
; SDAG-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
; SDAG-NEXT: s_mov_b32 s8, 2
; SDAG-NEXT: s_mov_b32 s9, s10
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: .LBB2_1: ; %loop_body
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; SDAG-NEXT: s_add_co_i32 s12, s9, 8
; SDAG-NEXT: s_wait_loadcnt 0x0
; SDAG-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v9, s12
; SDAG-NEXT: v_mov_b32_e32 v8, v3
; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: global_load_b32 v3, v0, s[4:5]
; SDAG-NEXT: global_load_b32 v4, v0, s[0:1]
; SDAG-NEXT: v_dual_add_nc_u32 v10, v5, v6 :: v_dual_mov_b32 v6, v2
; SDAG-NEXT: global_load_async_to_lds_b32 v9, v0, s[4:5] offset:4 nv
; SDAG-NEXT: v_mov_b32_e32 v9, s9
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: ; wait_asyncmark(2)
; SDAG-NEXT: s_wait_asynccnt 0x2
; SDAG-NEXT: s_add_co_i32 s8, s8, 1
; SDAG-NEXT: s_add_co_i32 s9, s9, 4
; SDAG-NEXT: ds_load_b32 v9, v9
; SDAG-NEXT: v_mov_b32_e32 v5, v1
; SDAG-NEXT: s_cmp_lt_i32 s8, s11
; SDAG-NEXT: s_wait_xcnt 0x0
; SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 4
; SDAG-NEXT: s_add_nc_u64 s[4:5], s[4:5], 4
; SDAG-NEXT: s_wait_dscnt 0x0
; SDAG-NEXT: v_add_nc_u32_e32 v9, v10, v9
; SDAG-NEXT: global_store_b32 v0, v9, s[6:7]
; SDAG-NEXT: s_wait_xcnt 0x0
; SDAG-NEXT: s_add_nc_u64 s[6:7], s[6:7], 4
; SDAG-NEXT: s_cbranch_scc1 .LBB2_1
; SDAG-NEXT: ; %bb.2: ; %epilog
; SDAG-NEXT: s_add_co_i32 s0, s11, -2
; SDAG-NEXT: ; wait_asyncmark(1)
; SDAG-NEXT: s_wait_asynccnt 0x1
; SDAG-NEXT: s_lshl2_add_u32 s1, s0, s10
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; SDAG-NEXT: v_dual_add_nc_u32 v2, v8, v7 :: v_dual_mov_b32 v0, s1
; SDAG-NEXT: ds_load_b32 v1, v0
; SDAG-NEXT: s_wait_dscnt 0x0
; SDAG-NEXT: v_dual_mov_b32 v5, s0 :: v_dual_add_nc_u32 v1, v2, v1
; SDAG-NEXT: global_store_b32 v5, v1, s[2:3] scale_offset
; SDAG-NEXT: ; wait_asyncmark(0)
; SDAG-NEXT: s_wait_asynccnt 0x0
; SDAG-NEXT: ds_load_b32 v0, v0 offset:4
; SDAG-NEXT: s_wait_loadcnt 0x0
; SDAG-NEXT: s_wait_xcnt 0x0
; SDAG-NEXT: v_add_nc_u32_e32 v1, v3, v4
; SDAG-NEXT: s_wait_dscnt 0x0
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; SDAG-NEXT: v_add_nc_u32_e32 v0, v1, v0
; SDAG-NEXT: global_store_b32 v5, v0, s[2:3] offset:4 scale_offset
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_pipelined_loop_with_global:
; GISEL: ; %bb.0: ; %prolog
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_clause 0x2
; GISEL-NEXT: s_load_b96 s[8:10], s[4:5], 0x24 nv
; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
; GISEL-NEXT: s_load_b32 s11, s[4:5], 0x44 nv
; GISEL-NEXT: v_mov_b32_e32 v0, 0
; GISEL-NEXT: s_wait_kmcnt 0x0
; GISEL-NEXT: s_load_b32 s14, s[8:9], 0x0
; GISEL-NEXT: s_load_b32 s15, s[0:1], 0x0
; GISEL-NEXT: v_mov_b32_e32 v1, s10
; GISEL-NEXT: s_add_co_u32 s6, s10, 4
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_dual_mov_b32 v4, 4 :: v_dual_mov_b32 v3, s6
; GISEL-NEXT: s_mov_b64 s[6:7], s[2:3]
; GISEL-NEXT: s_clause 0x2
; GISEL-NEXT: global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: global_load_b32 v1, v0, s[8:9] offset:4
; GISEL-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
; GISEL-NEXT: s_wait_xcnt 0x0
; GISEL-NEXT: s_add_co_u32 s0, s0, 8
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
; GISEL-NEXT: s_add_co_u32 s4, s8, 8
; GISEL-NEXT: s_add_co_ci_u32 s5, s9, 0
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v4, s[8:9] offset:4 nv
; GISEL-NEXT: s_wait_loadcnt 0x1
; GISEL-NEXT: v_readfirstlane_b32 s12, v1
; GISEL-NEXT: s_wait_loadcnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s13, v2
; GISEL-NEXT: s_mov_b32 s8, 2
; GISEL-NEXT: s_mov_b32 s9, s10
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: s_mov_b32 s17, s12
; GISEL-NEXT: s_mov_b32 s19, s13
; GISEL-NEXT: .LBB2_1: ; %loop_body
; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GISEL-NEXT: s_add_co_u32 s16, s9, 8
; GISEL-NEXT: s_clause 0x1
; GISEL-NEXT: global_load_b32 v1, v0, s[4:5]
; GISEL-NEXT: global_load_b32 v2, v0, s[0:1]
; GISEL-NEXT: v_mov_b32_e32 v3, s16
; GISEL-NEXT: s_mov_b32 s18, s17
; GISEL-NEXT: s_wait_kmcnt 0x0
; GISEL-NEXT: s_add_co_i32 s15, s14, s15
; GISEL-NEXT: s_add_co_i32 s8, s8, 1
; GISEL-NEXT: s_mov_b32 s16, s19
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v0, s[4:5] offset:4 nv
; GISEL-NEXT: v_mov_b32_e32 v3, s9
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(2)
; GISEL-NEXT: s_wait_asynccnt 0x2
; GISEL-NEXT: s_mov_b32 s14, s12
; GISEL-NEXT: ds_load_b32 v3, v3
; GISEL-NEXT: s_wait_dscnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s17, v3
; GISEL-NEXT: s_add_co_i32 s15, s15, s17
; GISEL-NEXT: s_wait_xcnt 0x0
; GISEL-NEXT: s_add_co_u32 s0, s0, 4
; GISEL-NEXT: v_mov_b32_e32 v3, s15
; GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
; GISEL-NEXT: s_add_co_u32 s4, s4, 4
; GISEL-NEXT: s_add_co_ci_u32 s5, s5, 0
; GISEL-NEXT: s_mov_b32 s15, s13
; GISEL-NEXT: global_store_b32 v0, v3, s[6:7]
; GISEL-NEXT: s_wait_xcnt 0x0
; GISEL-NEXT: s_add_co_u32 s6, s6, 4
; GISEL-NEXT: s_add_co_ci_u32 s7, s7, 0
; GISEL-NEXT: s_add_co_u32 s9, s9, 4
; GISEL-NEXT: s_cmp_lt_i32 s8, s11
; GISEL-NEXT: s_wait_loadcnt 0x1
; GISEL-NEXT: v_readfirstlane_b32 s17, v1
; GISEL-NEXT: s_wait_loadcnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s19, v2
; GISEL-NEXT: s_cbranch_scc1 .LBB2_1
; GISEL-NEXT: ; %bb.2: ; %epilog
; GISEL-NEXT: s_add_co_i32 s0, s11, -2
; GISEL-NEXT: ; wait_asyncmark(1)
; GISEL-NEXT: s_wait_asynccnt 0x1
; GISEL-NEXT: s_lshl_b32 s1, s0, 2
; GISEL-NEXT: s_add_co_i32 s4, s18, s16
; GISEL-NEXT: s_add_co_u32 s1, s10, s1
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_mov_b32_e32 v0, s1
; GISEL-NEXT: ds_load_b32 v1, v0
; GISEL-NEXT: s_wait_dscnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s1, v1
; GISEL-NEXT: v_mov_b32_e32 v1, s0
; GISEL-NEXT: s_add_co_i32 s1, s4, s1
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_mov_b32_e32 v2, s1
; GISEL-NEXT: s_add_co_i32 s1, s17, s19
; GISEL-NEXT: global_store_b32 v1, v2, s[2:3] scale_offset
; GISEL-NEXT: ; wait_asyncmark(0)
; GISEL-NEXT: s_wait_asynccnt 0x0
; GISEL-NEXT: ds_load_b32 v0, v0 offset:4
; GISEL-NEXT: s_wait_dscnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GISEL-NEXT: s_add_co_i32 s0, s1, s0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_mov_b32_e32 v0, s0
; GISEL-NEXT: global_store_b32 v1, v0, s[2:3] offset:4 scale_offset
; GISEL-NEXT: s_endpgm
prolog:
; Load first iteration
%v0 = load i32, ptr addrspace(1) %foo
%g0 = load i32, ptr addrspace(1) %bar
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20)
call void @llvm.amdgcn.asyncmark()
; Load second iteration
%foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
%v1 = load i32, ptr addrspace(1) %foo_gep1
%bar_gep1 = getelementptr i32, ptr addrspace(1) %bar, i32 1
%g1 = load i32, ptr addrspace(1) %bar_gep1
%lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep1, ptr addrspace(3) %lds_gep1, i32 4, i32 u0x20)
call void @llvm.amdgcn.asyncmark()
br label %loop_body
loop_body:
%i = phi i32 [ 2, %prolog ], [ %i.next, %loop_body ]
%prev_v = phi i32 [ %v0, %prolog ], [ %v1, %loop_body ]
%prev_g = phi i32 [ %g0, %prolog ], [ %g1, %loop_body ]
%v1_phi = phi i32 [ %v1, %prolog ], [ %cur_v, %loop_body ]
%g1_phi = phi i32 [ %g1, %prolog ], [ %cur_g, %loop_body ]
; Load next iteration
%foo_gep_cur = getelementptr i32, ptr addrspace(1) %foo, i32 %i
%cur_v = load i32, ptr addrspace(1) %foo_gep_cur
%bar_gep_cur = getelementptr i32, ptr addrspace(1) %bar, i32 %i
%cur_g = load i32, ptr addrspace(1) %bar_gep_cur
%lds_gep_cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i
call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep_cur, ptr addrspace(3) %lds_gep_cur, i32 4, i32 u0x20)
call void @llvm.amdgcn.asyncmark()
; Wait for iteration i-2 and process
call void @llvm.amdgcn.wait.asyncmark(i16 2)
%lds_idx = sub i32 %i, 2
%lds_gep_read = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_idx
%lds_val = load i32, ptr addrspace(3) %lds_gep_read
%sum1 = add i32 %prev_v, %prev_g
%sum2 = add i32 %sum1, %lds_val
%out_gep = getelementptr i32, ptr addrspace(1) %out, i32 %lds_idx
store i32 %sum2, ptr addrspace(1) %out_gep
%i.next = add i32 %i, 1
%cmp = icmp slt i32 %i.next, %n
br i1 %cmp, label %loop_body, label %epilog
epilog:
; Process remaining iterations
call void @llvm.amdgcn.wait.asyncmark(i16 1)
%lds_n_2 = sub i32 %n, 2
%lds_gep_n_2 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_2
%lds_val_n_2 = load i32, ptr addrspace(3) %lds_gep_n_2
%sum_e1 = add i32 %v1_phi, %g1_phi
%sum_e2 = add i32 %sum_e1, %lds_val_n_2
%out_gep_e1 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_2
store i32 %sum_e2, ptr addrspace(1) %out_gep_e1
call void @llvm.amdgcn.wait.asyncmark(i16 0)
%lds_n_1 = sub i32 %n, 1
%lds_gep_n_1 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_1
%lds_val_n_1 = load i32, ptr addrspace(3) %lds_gep_n_1
%sum_e3 = add i32 %cur_v, %cur_g
%sum_e4 = add i32 %sum_e3, %lds_val_n_1
%out_gep_e2 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_1
store i32 %sum_e4, ptr addrspace(1) %out_gep_e2
ret void
}