llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck %s -check-prefixes=SDAG
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 < %s | FileCheck %s -check-prefixes=GISEL

; Test async mark/wait with global_load_lds and global loads
; This version uses wave barriers to enforce program order so that unrelated vmem
; instructions do not get reordered before reaching this point.

define void @interleaved_with_wave_barrier(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out) {
; SDAG-LABEL: interleaved_with_wave_barrier:
; SDAG:       ; %bb.0: ; %entry
; SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
; SDAG-NEXT:    s_wait_kmcnt 0x0
; SDAG-NEXT:    v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v9, v4
; SDAG-NEXT:    v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v6, v5
; SDAG-NEXT:    v_add_nc_u64_e32 v[4:5], 0x54, v[0:1]
; SDAG-NEXT:    global_load_b32 v12, v[8:9], off offset:44
; SDAG-NEXT:    global_load_b32 v13, v[0:1], off offset:4
; SDAG-NEXT:    v_add_nc_u32_e32 v3, 0x54, v2
; SDAG-NEXT:    v_add_nc_u64_e32 v[10:11], 0x58, v[8:9]
; SDAG-NEXT:    ; wave barrier
; SDAG-NEXT:    s_mov_b32 s0, 0x58
; SDAG-NEXT:    s_wait_loadcnt 0x0
; SDAG-NEXT:    v_dual_add_nc_u32 v14, s0, v2 :: v_dual_add_nc_u32 v12, v13, v12
; SDAG-NEXT:    global_load_async_to_lds_b32 v3, v[4:5], off offset:4 th:TH_LOAD_NT nv
; SDAG-NEXT:    ; wave barrier
; SDAG-NEXT:    ; asyncmark
; SDAG-NEXT:    global_load_b32 v15, v[0:1], off offset:8
; SDAG-NEXT:    ; wave barrier
; SDAG-NEXT:    global_load_async_to_lds_b32 v14, v[10:11], off offset:4 th:TH_LOAD_LU nv
; SDAG-NEXT:    ; wave barrier
; SDAG-NEXT:    global_load_b32 v16, v[8:9], off offset:48
; SDAG-NEXT:    ; asyncmark
; SDAG-NEXT:    ; wait_asyncmark(1)
; SDAG-NEXT:    s_wait_asynccnt 0x1
; SDAG-NEXT:    ds_load_b32 v17, v2 offset:84
; SDAG-NEXT:    ; wait_asyncmark(0)
; SDAG-NEXT:    s_wait_asynccnt 0x0
; SDAG-NEXT:    ds_load_b32 v2, v2 offset:88
; SDAG-NEXT:    s_wait_loadcnt_dscnt 0x101
; SDAG-NEXT:    v_add3_u32 v12, v12, v17, v15
; SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
; SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; SDAG-NEXT:    v_add3_u32 v2, v12, v16, v2
; SDAG-NEXT:    global_store_b32 v[6:7], v2, off
; SDAG-NEXT:    s_set_pc_i64 s[30:31]
;
; GISEL-LABEL: interleaved_with_wave_barrier:
; GISEL:       ; %bb.0: ; %entry
; GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
; GISEL-NEXT:    s_wait_kmcnt 0x0
; GISEL-NEXT:    v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
; GISEL-NEXT:    v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
; GISEL-NEXT:    v_add_co_u32 v6, vcc_lo, v0, 0x54
; GISEL-NEXT:    global_load_b32 v12, v[8:9], off offset:44
; GISEL-NEXT:    global_load_b32 v13, v[0:1], off offset:4
; GISEL-NEXT:    v_add_nc_u32_e32 v3, 0x54, v2
; GISEL-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
; GISEL-NEXT:    v_add_co_u32 v10, vcc_lo, v8, 0x58
; GISEL-NEXT:    ; wave barrier
; GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GISEL-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v9, vcc_lo
; GISEL-NEXT:    s_mov_b32 s0, 0x58
; GISEL-NEXT:    s_wait_loadcnt 0x0
; GISEL-NEXT:    v_dual_add_nc_u32 v14, s0, v2 :: v_dual_add_nc_u32 v12, v13, v12
; GISEL-NEXT:    global_load_async_to_lds_b32 v3, v[6:7], off offset:4 th:TH_LOAD_NT nv
; GISEL-NEXT:    ; wave barrier
; GISEL-NEXT:    ; asyncmark
; GISEL-NEXT:    global_load_b32 v15, v[0:1], off offset:8
; GISEL-NEXT:    ; wave barrier
; GISEL-NEXT:    global_load_async_to_lds_b32 v14, v[10:11], off offset:4 th:TH_LOAD_LU nv
; GISEL-NEXT:    ; wave barrier
; GISEL-NEXT:    global_load_b32 v16, v[8:9], off offset:48
; GISEL-NEXT:    ; asyncmark
; GISEL-NEXT:    ; wait_asyncmark(1)
; GISEL-NEXT:    s_wait_asynccnt 0x1
; GISEL-NEXT:    ds_load_b32 v17, v2 offset:84
; GISEL-NEXT:    ; wait_asyncmark(0)
; GISEL-NEXT:    s_wait_asynccnt 0x0
; GISEL-NEXT:    ds_load_b32 v2, v2 offset:88
; GISEL-NEXT:    s_wait_loadcnt_dscnt 0x101
; GISEL-NEXT:    v_add3_u32 v12, v12, v17, v15
; GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
; GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GISEL-NEXT:    v_add3_u32 v2, v12, v16, v2
; GISEL-NEXT:    global_store_b32 v[4:5], v2, off
; GISEL-NEXT:    s_set_pc_i64 s[30:31]
entry:
  ; First batch: global load, global load, async global-to-LDS
  %bar_gep11 = getelementptr i32, ptr addrspace(1) %bar, i32 11
  %bar_v11 = load i32, ptr addrspace(1) %bar_gep11
  %foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
  %foo_v1 = load i32, ptr addrspace(1) %foo_gep1
  %lds_gep21 = getelementptr i32, ptr addrspace(3) %lds, i32 21
  %bar_gep21 = getelementptr i32, ptr addrspace(1) %foo, i32 21
  call void @llvm.amdgcn.wave.barrier()
  call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar_gep21, ptr addrspace(3) %lds_gep21, i32 4, i32 u0x21)
  call void @llvm.amdgcn.wave.barrier()
  call void @llvm.amdgcn.asyncmark()

  ; Second batch: global load, async global-to-LDS, global load
  %foo_gep2 = getelementptr i32, ptr addrspace(1) %foo, i32 2
  %foo_v2 = load i32, ptr addrspace(1) %foo_gep2
  %bar_gep22 = getelementptr i32, ptr addrspace(1) %bar, i32 22
  %lds_gep22 = getelementptr i32, ptr addrspace(3) %lds, i32 22
  call void @llvm.amdgcn.wave.barrier()
  call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar_gep22, ptr addrspace(3) %lds_gep22, i32 4, i32 u0x23)
  call void @llvm.amdgcn.wave.barrier()
  %bar_gep12 = getelementptr i32, ptr addrspace(1) %bar, i32 12
  %bar_v12 = load i32, ptr addrspace(1) %bar_gep12
  call void @llvm.amdgcn.asyncmark()

  ; Wait for first async mark and read from LDS
  call void @llvm.amdgcn.wait.asyncmark(i16 1)
  %lds_val21 = load i32, ptr addrspace(3) %lds_gep21

  ; Wait for the next async mark.
  ; Notable that the asyncmark is sufficient to prevent the optimizer from coalescing the previous ds_load with the next one.
  call void @llvm.amdgcn.wait.asyncmark(i16 0)
  %lds_val22 = load i32, ptr addrspace(3) %lds_gep22
  %sum1 = add i32 %foo_v1, %bar_v11
  %sum2 = add i32 %sum1, %lds_val21
  %sum3 = add i32 %sum2, %foo_v2
  ; Finally a loadcnt(0) for %bar_v12, which was not included in the async mark that followed it.
  %sum4 = add i32 %sum3, %bar_v12
  %sum5 = add i32 %sum4, %lds_val22
  store i32 %sum5, ptr addrspace(1) %out

  ret void
}

; A perfect loop that is unlikely to exist in real life. It uses only async
; operations, and result in waits that exactly match the stream of
; those outstanding operations.

define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) {
; SDAG-LABEL: test_pipelined_loop:
; SDAG:       ; %bb.0: ; %prolog
; SDAG-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-NEXT:    v_nop
; SDAG-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-NEXT:    s_clause 0x1
; SDAG-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24 nv
; SDAG-NEXT:    s_load_b32 s3, s[4:5], 0x44 nv
; SDAG-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; SDAG-NEXT:    s_wait_kmcnt 0x0
; SDAG-NEXT:    s_add_co_i32 s6, s2, 4
; SDAG-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, 4
; SDAG-NEXT:    v_mov_b32_e32 v4, s6
; SDAG-NEXT:    s_mov_b32 s6, 2
; SDAG-NEXT:    s_mov_b32 s7, s2
; SDAG-NEXT:    global_load_async_to_lds_b32 v2, v0, s[0:1] offset:4 nv
; SDAG-NEXT:    ; asyncmark
; SDAG-NEXT:    global_load_async_to_lds_b32 v4, v3, s[0:1] offset:4 nv
; SDAG-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 8
; SDAG-NEXT:    ; asyncmark
; SDAG-NEXT:  .LBB1_1: ; %loop_body
; SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT:    s_add_co_i32 s8, s7, 8
; SDAG-NEXT:    s_add_co_i32 s6, s6, 1
; SDAG-NEXT:    v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v3, s7
; SDAG-NEXT:    s_add_co_i32 s7, s7, 4
; SDAG-NEXT:    s_cmp_lt_i32 s6, s3
; SDAG-NEXT:    global_load_async_to_lds_b32 v2, v0, s[0:1] offset:4 nv
; SDAG-NEXT:    ; asyncmark
; SDAG-NEXT:    ; wait_asyncmark(2)
; SDAG-NEXT:    s_wait_asynccnt 0x2
; SDAG-NEXT:    ds_load_b32 v3, v3
; SDAG-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 4
; SDAG-NEXT:    s_wait_dscnt 0x0
; SDAG-NEXT:    v_add_nc_u32_e32 v1, v1, v3
; SDAG-NEXT:    s_cbranch_scc1 .LBB1_1
; SDAG-NEXT:  ; %bb.2: ; %epilog
; SDAG-NEXT:    s_lshl2_add_u32 s0, s3, s2
; SDAG-NEXT:    ; wait_asyncmark(1)
; SDAG-NEXT:    s_wait_asynccnt 0x1
; SDAG-NEXT:    s_add_co_i32 s0, s0, -8
; SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s0
; SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34 nv
; SDAG-NEXT:    ds_load_b32 v0, v0
; SDAG-NEXT:    ; wait_asyncmark(0)
; SDAG-NEXT:    s_wait_dscnt 0x0
; SDAG-NEXT:    s_wait_asynccnt 0x0
; SDAG-NEXT:    v_add_nc_u32_e32 v0, v1, v0
; SDAG-NEXT:    s_wait_kmcnt 0x0
; SDAG-NEXT:    global_store_b32 v2, v0, s[0:1]
; SDAG-NEXT:    s_endpgm
;
; GISEL-LABEL: test_pipelined_loop:
; GISEL:       ; %bb.0: ; %prolog
; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT:    s_mov_b64 s[64:65], 0
; GISEL-NEXT:    v_nop
; GISEL-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GISEL-NEXT:    s_clause 0x1
; GISEL-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24 nv
; GISEL-NEXT:    s_load_b32 s3, s[4:5], 0x44 nv
; GISEL-NEXT:    v_mov_b32_e32 v0, 0
; GISEL-NEXT:    s_mov_b32 s7, 2
; GISEL-NEXT:    s_wait_kmcnt 0x0
; GISEL-NEXT:    s_add_co_u32 s6, s2, 4
; GISEL-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, 4
; GISEL-NEXT:    v_mov_b32_e32 v3, s6
; GISEL-NEXT:    s_mov_b32 s6, 0
; GISEL-NEXT:    s_mov_b32 s8, s2
; GISEL-NEXT:    global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
; GISEL-NEXT:    ; asyncmark
; GISEL-NEXT:    global_load_async_to_lds_b32 v3, v2, s[0:1] offset:4 nv
; GISEL-NEXT:    s_add_co_u32 s0, s0, 8
; GISEL-NEXT:    s_add_co_ci_u32 s1, s1, 0
; GISEL-NEXT:    ; asyncmark
; GISEL-NEXT:  .LBB1_1: ; %loop_body
; GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GISEL-NEXT:    s_add_co_u32 s9, s8, 8
; GISEL-NEXT:    s_add_co_i32 s7, s7, 1
; GISEL-NEXT:    v_dual_mov_b32 v1, s9 :: v_dual_mov_b32 v2, s8
; GISEL-NEXT:    global_load_async_to_lds_b32 v1, v0, s[0:1] offset:4 nv
; GISEL-NEXT:    ; asyncmark
; GISEL-NEXT:    ; wait_asyncmark(2)
; GISEL-NEXT:    s_wait_asynccnt 0x2
; GISEL-NEXT:    ds_load_b32 v2, v2
; GISEL-NEXT:    s_wait_dscnt 0x0
; GISEL-NEXT:    v_readfirstlane_b32 s9, v2
; GISEL-NEXT:    s_add_co_i32 s6, s6, s9
; GISEL-NEXT:    s_add_co_u32 s0, s0, 4
; GISEL-NEXT:    s_add_co_ci_u32 s1, s1, 0
; GISEL-NEXT:    s_add_co_u32 s8, s8, 4
; GISEL-NEXT:    s_cmp_lt_i32 s7, s3
; GISEL-NEXT:    s_cbranch_scc1 .LBB1_1
; GISEL-NEXT:  ; %bb.2: ; %epilog
; GISEL-NEXT:    s_lshl_b32 s0, s3, 2
; GISEL-NEXT:    ; wait_asyncmark(1)
; GISEL-NEXT:    s_wait_asynccnt 0x1
; GISEL-NEXT:    s_add_co_u32 s0, s2, s0
; GISEL-NEXT:    v_mov_b32_e32 v1, 0
; GISEL-NEXT:    s_add_co_u32 s0, s0, -8
; GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT:    v_mov_b32_e32 v0, s0
; GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34 nv
; GISEL-NEXT:    ds_load_b32 v0, v0
; GISEL-NEXT:    ; wait_asyncmark(0)
; GISEL-NEXT:    s_wait_dscnt 0x0
; GISEL-NEXT:    s_wait_asynccnt 0x0
; GISEL-NEXT:    v_readfirstlane_b32 s2, v0
; GISEL-NEXT:    s_add_co_i32 s2, s6, s2
; GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT:    v_mov_b32_e32 v0, s2
; GISEL-NEXT:    s_wait_kmcnt 0x0
; GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT:    s_endpgm
prolog:
  ; Load first iteration
  call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20)
  call void @llvm.amdgcn.asyncmark()

  ; Load second iteration
  %lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
  %foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
  call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep1, ptr addrspace(3) %lds_gep1, i32 4, i32 u0x20)
  call void @llvm.amdgcn.asyncmark()

  br label %loop_body

loop_body:
  %i = phi i32 [ 2, %prolog ], [ %i.next, %loop_body ]
  %sum = phi i32 [ 0, %prolog ], [ %sum_i, %loop_body ]

  ; Load next iteration
  %lds_gep_cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i
  %foo_gep_cur = getelementptr i32, ptr addrspace(1) %foo, i32 %i
  call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep_cur, ptr addrspace(3) %lds_gep_cur, i32 4, i32 u0x20)
  call void @llvm.amdgcn.asyncmark()

  ; Wait for iteration i-2 and process
  call void @llvm.amdgcn.wait.asyncmark(i16 2)
  %lds_idx = sub i32 %i, 2
  %lds_gep_read = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_idx
  %lds_val = load i32, ptr addrspace(3) %lds_gep_read

  %sum_i = add i32 %sum, %lds_val

  %i.next = add i32 %i, 1
  %cmp = icmp slt i32 %i.next, %n
  br i1 %cmp, label %loop_body, label %epilog

epilog:
  ; Process remaining iterations
  call void @llvm.amdgcn.wait.asyncmark(i16 1)
  %lds_n_2 = sub i32 %n, 2
  %lds_gep_n_2 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_2
  %lds_val_n_2 = load i32, ptr addrspace(3) %lds_gep_n_2
  %sum_e2 = add i32 %sum_i, %lds_val_n_2
  %out_gep_e1 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_2

  call void @llvm.amdgcn.wait.asyncmark(i16 0)
  %lds_n_1 = sub i32 %n, 1
  %lds_gep_n_1 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_1
  %lds_val_n_1 = load i32, ptr addrspace(3) %lds_gep_n_1
  %sum_e1 = add i32 %sum_e2, %lds_val_n_1
  store i32 %sum_e2, ptr addrspace(1) %bar

  ret void
}

; Software pipelined loop with async global-to-LDS and global loads

define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) {
; SDAG-LABEL: test_pipelined_loop_with_global:
; SDAG:       ; %bb.0: ; %prolog
; SDAG-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-NEXT:    s_mov_b64 s[64:65], 0
; SDAG-NEXT:    v_nop
; SDAG-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; SDAG-NEXT:    s_clause 0x1
; SDAG-NEXT:    s_load_b96 s[8:10], s[4:5], 0x24 nv
; SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34 nv
; SDAG-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v7, 4
; SDAG-NEXT:    s_load_b32 s11, s[4:5], 0x44 nv
; SDAG-NEXT:    s_wait_kmcnt 0x0
; SDAG-NEXT:    s_add_co_i32 s7, s10, 4
; SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; SDAG-NEXT:    v_dual_mov_b32 v8, s10 :: v_dual_mov_b32 v9, s7
; SDAG-NEXT:    s_load_b32 s6, s[8:9], 0x0
; SDAG-NEXT:    s_load_b32 s12, s[0:1], 0x0
; SDAG-NEXT:    s_add_nc_u64 s[4:5], s[8:9], 8
; SDAG-NEXT:    global_load_async_to_lds_b32 v8, v0, s[8:9] offset:4 nv
; SDAG-NEXT:    ; asyncmark
; SDAG-NEXT:    s_clause 0x1
; SDAG-NEXT:    global_load_b32 v1, v0, s[8:9] offset:4
; SDAG-NEXT:    global_load_b32 v2, v0, s[0:1] offset:4
; SDAG-NEXT:    global_load_async_to_lds_b32 v9, v7, s[8:9] offset:4 nv
; SDAG-NEXT:    s_wait_xcnt 0x0
; SDAG-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 8
; SDAG-NEXT:    s_mov_b32 s8, 2
; SDAG-NEXT:    s_mov_b32 s9, s10
; SDAG-NEXT:    ; asyncmark
; SDAG-NEXT:    s_wait_kmcnt 0x0
; SDAG-NEXT:    v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v6, s12
; SDAG-NEXT:    s_mov_b64 s[6:7], s[2:3]
; SDAG-NEXT:    s_wait_loadcnt 0x0
; SDAG-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
; SDAG-NEXT:  .LBB2_1: ; %loop_body
; SDAG-NEXT:    ; =>This Inner Loop Header: Depth=1
; SDAG-NEXT:    s_add_co_i32 s12, s9, 8
; SDAG-NEXT:    s_wait_loadcnt 0x0
; SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; SDAG-NEXT:    v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v9, s12
; SDAG-NEXT:    v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v10, s9
; SDAG-NEXT:    global_load_b32 v3, v0, s[4:5]
; SDAG-NEXT:    v_dual_mov_b32 v6, v2 :: v_dual_add_nc_u32 v11, v5, v6
; SDAG-NEXT:    global_load_async_to_lds_b32 v9, v0, s[4:5] offset:4 nv
; SDAG-NEXT:    global_load_b32 v4, v0, s[0:1]
; SDAG-NEXT:    ; asyncmark
; SDAG-NEXT:    ; wait_asyncmark(2)
; SDAG-NEXT:    s_wait_asynccnt 0x2
; SDAG-NEXT:    ds_load_b32 v10, v10
; SDAG-NEXT:    v_mov_b32_e32 v5, v1
; SDAG-NEXT:    s_add_co_i32 s8, s8, 1
; SDAG-NEXT:    s_add_co_i32 s9, s9, 4
; SDAG-NEXT:    s_cmp_lt_i32 s8, s11
; SDAG-NEXT:    s_wait_xcnt 0x1
; SDAG-NEXT:    s_add_nc_u64 s[4:5], s[4:5], 4
; SDAG-NEXT:    s_wait_xcnt 0x0
; SDAG-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 4
; SDAG-NEXT:    s_wait_dscnt 0x0
; SDAG-NEXT:    v_add_nc_u32_e32 v10, v11, v10
; SDAG-NEXT:    global_store_b32 v0, v10, s[6:7]
; SDAG-NEXT:    s_wait_xcnt 0x0
; SDAG-NEXT:    s_add_nc_u64 s[6:7], s[6:7], 4
; SDAG-NEXT:    s_cbranch_scc1 .LBB2_1
; SDAG-NEXT:  ; %bb.2: ; %epilog
; SDAG-NEXT:    s_add_co_i32 s0, s11, -2
; SDAG-NEXT:    ; wait_asyncmark(1)
; SDAG-NEXT:    s_wait_asynccnt 0x1
; SDAG-NEXT:    s_lshl2_add_u32 s1, s0, s10
; SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; SDAG-NEXT:    v_dual_add_nc_u32 v2, v8, v7 :: v_dual_mov_b32 v0, s1
; SDAG-NEXT:    ds_load_b32 v1, v0
; SDAG-NEXT:    v_mov_b32_e32 v5, s0
; SDAG-NEXT:    s_add_co_i32 s0, s11, -1
; SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
; SDAG-NEXT:    v_dual_add_nc_u32 v1, v2, v1 :: v_dual_add_nc_u32 v2, v3, v4
; SDAG-NEXT:    global_store_b32 v5, v1, s[2:3] scale_offset
; SDAG-NEXT:    ; wait_asyncmark(0)
; SDAG-NEXT:    s_wait_asynccnt 0x0
; SDAG-NEXT:    ds_load_b32 v0, v0 offset:4
; SDAG-NEXT:    s_wait_dscnt 0x0
; SDAG-NEXT:    v_dual_add_nc_u32 v0, v2, v0 :: v_dual_mov_b32 v2, s0
; SDAG-NEXT:    global_store_b32 v2, v0, s[2:3] scale_offset
; SDAG-NEXT:    s_endpgm
;
; GISEL-LABEL: test_pipelined_loop_with_global:
; GISEL:       ; %bb.0: ; %prolog
; GISEL-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT:    s_mov_b64 s[64:65], 0
; GISEL-NEXT:    v_nop
; GISEL-NEXT:    global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GISEL-NEXT:    s_clause 0x1
; GISEL-NEXT:    s_load_b96 s[8:10], s[4:5], 0x24 nv
; GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34 nv
; GISEL-NEXT:    v_mov_b32_e32 v0, 0
; GISEL-NEXT:    s_load_b32 s11, s[4:5], 0x44 nv
; GISEL-NEXT:    s_mov_b32 s12, 2
; GISEL-NEXT:    s_wait_kmcnt 0x0
; GISEL-NEXT:    s_load_b32 s13, s[8:9], 0x0
; GISEL-NEXT:    s_load_b32 s16, s[0:1], 0x0
; GISEL-NEXT:    v_mov_b32_e32 v1, s10
; GISEL-NEXT:    s_add_co_u32 s6, s10, 4
; GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT:    v_dual_mov_b32 v4, 4 :: v_dual_mov_b32 v5, s6
; GISEL-NEXT:    s_add_co_u32 s6, s8, 8
; GISEL-NEXT:    global_load_async_to_lds_b32 v1, v0, s[8:9] offset:4 nv
; GISEL-NEXT:    ; asyncmark
; GISEL-NEXT:    s_clause 0x1
; GISEL-NEXT:    global_load_b32 v2, v0, s[8:9] offset:4
; GISEL-NEXT:    global_load_b32 v3, v0, s[0:1] offset:4
; GISEL-NEXT:    global_load_async_to_lds_b32 v5, v4, s[8:9] offset:4 nv
; GISEL-NEXT:    s_add_co_ci_u32 s7, s9, 0
; GISEL-NEXT:    s_wait_xcnt 0x0
; GISEL-NEXT:    s_add_co_u32 s0, s0, 8
; GISEL-NEXT:    s_mov_b32 s14, s10
; GISEL-NEXT:    s_mov_b64 s[4:5], s[2:3]
; GISEL-NEXT:    s_add_co_ci_u32 s1, s1, 0
; GISEL-NEXT:    ; asyncmark
; GISEL-NEXT:    s_wait_loadcnt 0x1
; GISEL-NEXT:    v_readfirstlane_b32 s8, v2
; GISEL-NEXT:    s_wait_loadcnt 0x0
; GISEL-NEXT:    v_readfirstlane_b32 s15, v3
; GISEL-NEXT:    s_mov_b32 s9, s8
; GISEL-NEXT:    s_mov_b32 s19, s15
; GISEL-NEXT:  .LBB2_1: ; %loop_body
; GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
; GISEL-NEXT:    s_add_co_u32 s17, s14, 8
; GISEL-NEXT:    global_load_b32 v2, v0, s[6:7]
; GISEL-NEXT:    v_dual_mov_b32 v1, s17 :: v_dual_mov_b32 v4, s14
; GISEL-NEXT:    s_mov_b32 s18, s9
; GISEL-NEXT:    s_wait_kmcnt 0x0
; GISEL-NEXT:    s_add_co_i32 s9, s13, s16
; GISEL-NEXT:    s_add_co_i32 s12, s12, 1
; GISEL-NEXT:    global_load_async_to_lds_b32 v1, v0, s[6:7] offset:4 nv
; GISEL-NEXT:    global_load_b32 v3, v0, s[0:1]
; GISEL-NEXT:    ; asyncmark
; GISEL-NEXT:    ; wait_asyncmark(2)
; GISEL-NEXT:    s_wait_asynccnt 0x2
; GISEL-NEXT:    ds_load_b32 v4, v4
; GISEL-NEXT:    s_mov_b32 s17, s19
; GISEL-NEXT:    s_mov_b32 s13, s8
; GISEL-NEXT:    s_wait_dscnt 0x0
; GISEL-NEXT:    v_readfirstlane_b32 s16, v4
; GISEL-NEXT:    s_add_co_i32 s9, s9, s16
; GISEL-NEXT:    s_wait_xcnt 0x1
; GISEL-NEXT:    s_add_co_u32 s6, s6, 4
; GISEL-NEXT:    v_mov_b32_e32 v4, s9
; GISEL-NEXT:    s_add_co_ci_u32 s7, s7, 0
; GISEL-NEXT:    s_wait_xcnt 0x0
; GISEL-NEXT:    s_add_co_u32 s0, s0, 4
; GISEL-NEXT:    s_add_co_ci_u32 s1, s1, 0
; GISEL-NEXT:    s_mov_b32 s16, s15
; GISEL-NEXT:    global_store_b32 v0, v4, s[4:5]
; GISEL-NEXT:    s_wait_xcnt 0x0
; GISEL-NEXT:    s_add_co_u32 s4, s4, 4
; GISEL-NEXT:    s_add_co_ci_u32 s5, s5, 0
; GISEL-NEXT:    s_add_co_u32 s14, s14, 4
; GISEL-NEXT:    s_cmp_lt_i32 s12, s11
; GISEL-NEXT:    s_wait_loadcnt 0x1
; GISEL-NEXT:    v_readfirstlane_b32 s9, v2
; GISEL-NEXT:    s_wait_loadcnt 0x0
; GISEL-NEXT:    v_readfirstlane_b32 s19, v3
; GISEL-NEXT:    s_cbranch_scc1 .LBB2_1
; GISEL-NEXT:  ; %bb.2: ; %epilog
; GISEL-NEXT:    s_add_co_i32 s0, s11, -2
; GISEL-NEXT:    ; wait_asyncmark(1)
; GISEL-NEXT:    s_wait_asynccnt 0x1
; GISEL-NEXT:    s_lshl_b32 s1, s0, 2
; GISEL-NEXT:    s_add_co_i32 s4, s18, s17
; GISEL-NEXT:    s_add_co_u32 s1, s10, s1
; GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT:    v_mov_b32_e32 v0, s1
; GISEL-NEXT:    ds_load_b32 v1, v0
; GISEL-NEXT:    s_wait_dscnt 0x0
; GISEL-NEXT:    v_readfirstlane_b32 s1, v1
; GISEL-NEXT:    v_mov_b32_e32 v1, s0
; GISEL-NEXT:    s_add_co_i32 s1, s4, s1
; GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT:    v_mov_b32_e32 v2, s1
; GISEL-NEXT:    s_add_co_i32 s1, s9, s19
; GISEL-NEXT:    global_store_b32 v1, v2, s[2:3] scale_offset
; GISEL-NEXT:    ; wait_asyncmark(0)
; GISEL-NEXT:    s_wait_asynccnt 0x0
; GISEL-NEXT:    ds_load_b32 v0, v0 offset:4
; GISEL-NEXT:    s_wait_dscnt 0x0
; GISEL-NEXT:    v_readfirstlane_b32 s0, v0
; GISEL-NEXT:    s_add_co_i32 s0, s1, s0
; GISEL-NEXT:    s_add_co_i32 s1, s11, -1
; GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s1
; GISEL-NEXT:    global_store_b32 v3, v0, s[2:3] scale_offset
; GISEL-NEXT:    s_endpgm
prolog:
  ; Load first iteration
  %v0 = load i32, ptr addrspace(1) %foo
  %g0 = load i32, ptr addrspace(1) %bar
  call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo, ptr addrspace(3) %lds, i32 4, i32 u0x20)
  call void @llvm.amdgcn.asyncmark()

  ; Load second iteration
  %foo_gep1 = getelementptr i32, ptr addrspace(1) %foo, i32 1
  %v1 = load i32, ptr addrspace(1) %foo_gep1
  %bar_gep1 = getelementptr i32, ptr addrspace(1) %bar, i32 1
  %g1 = load i32, ptr addrspace(1) %bar_gep1

  %lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
  call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep1, ptr addrspace(3) %lds_gep1, i32 4, i32 u0x20)
  call void @llvm.amdgcn.asyncmark()

  br label %loop_body

loop_body:
  %i = phi i32 [ 2, %prolog ], [ %i.next, %loop_body ]
  %prev_v = phi i32 [ %v0, %prolog ], [ %v1, %loop_body ]
  %prev_g = phi i32 [ %g0, %prolog ], [ %g1, %loop_body ]
  %v1_phi = phi i32 [ %v1, %prolog ], [ %cur_v, %loop_body ]
  %g1_phi = phi i32 [ %g1, %prolog ], [ %cur_g, %loop_body ]

  ; Load next iteration
  %foo_gep_cur = getelementptr i32, ptr addrspace(1) %foo, i32 %i
  %cur_v = load i32, ptr addrspace(1) %foo_gep_cur
  %bar_gep_cur = getelementptr i32, ptr addrspace(1) %bar, i32 %i
  %cur_g = load i32, ptr addrspace(1) %bar_gep_cur
  %lds_gep_cur = getelementptr i32, ptr addrspace(3) %lds, i32 %i
  call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %foo_gep_cur, ptr addrspace(3) %lds_gep_cur, i32 4, i32 u0x20)
  call void @llvm.amdgcn.asyncmark()

  ; Wait for iteration i-2 and process
  call void @llvm.amdgcn.wait.asyncmark(i16 2)
  %lds_idx = sub i32 %i, 2
  %lds_gep_read = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_idx
  %lds_val = load i32, ptr addrspace(3) %lds_gep_read

  %sum1 = add i32 %prev_v, %prev_g
  %sum2 = add i32 %sum1, %lds_val
  %out_gep = getelementptr i32, ptr addrspace(1) %out, i32 %lds_idx
  store i32 %sum2, ptr addrspace(1) %out_gep

  %i.next = add i32 %i, 1
  %cmp = icmp slt i32 %i.next, %n
  br i1 %cmp, label %loop_body, label %epilog

epilog:
  ; Process remaining iterations
  call void @llvm.amdgcn.wait.asyncmark(i16 1)
  %lds_n_2 = sub i32 %n, 2
  %lds_gep_n_2 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_2
  %lds_val_n_2 = load i32, ptr addrspace(3) %lds_gep_n_2
  %sum_e1 = add i32 %v1_phi, %g1_phi
  %sum_e2 = add i32 %sum_e1, %lds_val_n_2
  %out_gep_e1 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_2
  store i32 %sum_e2, ptr addrspace(1) %out_gep_e1

  call void @llvm.amdgcn.wait.asyncmark(i16 0)
  %lds_n_1 = sub i32 %n, 1
  %lds_gep_n_1 = getelementptr i32, ptr addrspace(3) %lds, i32 %lds_n_1
  %lds_val_n_1 = load i32, ptr addrspace(3) %lds_gep_n_1
  %sum_e3 = add i32 %cur_v, %cur_g
  %sum_e4 = add i32 %sum_e3, %lds_val_n_1
  %out_gep_e2 = getelementptr i32, ptr addrspace(1) %out, i32 %lds_n_1
  store i32 %sum_e4, ptr addrspace(1) %out_gep_e2

  ret void
}

; The second asyncmark's snapshot of the counters should be the same as the first;
; when wait.asyncmark(0) is called it should insert a asynccnt before the ds_load.

define void @consecutive_asyncmarks(ptr addrspace(1) %bar, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
; SDAG-LABEL: consecutive_asyncmarks:
; SDAG:       ; %bb.0: ; %entry
; SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
; SDAG-NEXT:    s_wait_kmcnt 0x0
; SDAG-NEXT:    global_load_async_to_lds_b32 v2, v[0:1], off offset:4
; SDAG-NEXT:    v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; SDAG-NEXT:    ; asyncmark
; SDAG-NEXT:    ; asyncmark
; SDAG-NEXT:    ; wait_asyncmark(0)
; SDAG-NEXT:    s_wait_asynccnt 0x0
; SDAG-NEXT:    ds_load_b32 v3, v2
; SDAG-NEXT:    s_wait_dscnt 0x0
; SDAG-NEXT:    global_store_b32 v[4:5], v3, off
; SDAG-NEXT:    s_set_pc_i64 s[30:31]
;
; GISEL-LABEL: consecutive_asyncmarks:
; GISEL:       ; %bb.0: ; %entry
; GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
; GISEL-NEXT:    s_wait_kmcnt 0x0
; GISEL-NEXT:    global_load_async_to_lds_b32 v2, v[0:1], off offset:4
; GISEL-NEXT:    v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GISEL-NEXT:    ; asyncmark
; GISEL-NEXT:    ; asyncmark
; GISEL-NEXT:    ; wait_asyncmark(0)
; GISEL-NEXT:    s_wait_asynccnt 0x0
; GISEL-NEXT:    ds_load_b32 v3, v2
; GISEL-NEXT:    s_wait_dscnt 0x0
; GISEL-NEXT:    global_store_b32 v[6:7], v3, off
; GISEL-NEXT:    s_set_pc_i64 s[30:31]
entry:
  call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0)
  call void @llvm.amdgcn.asyncmark()
  call void @llvm.amdgcn.asyncmark()
  call void @llvm.amdgcn.wait.asyncmark(i16 0)
  %val = load i32, ptr addrspace(3) %lds
  store i32 %val, ptr addrspace(1) %out
  ret void
}

; Similar to the previous test, except wait.asyncmark(1) should remove the
; the first two marks, allowing the second DMA to remain in flight (ie.
; asynccnt 0x1) during the ds_load.

define void @consecutive_asyncmarks_wait1(ptr addrspace(1) %bar, ptr addrspace(3) %lds, ptr addrspace(1) %out) {
; SDAG-LABEL: consecutive_asyncmarks_wait1:
; SDAG:       ; %bb.0: ; %entry
; SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
; SDAG-NEXT:    s_wait_kmcnt 0x0
; SDAG-NEXT:    v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; SDAG-NEXT:    v_add_nc_u32_e32 v3, 4, v2
; SDAG-NEXT:    s_clause 0x1
; SDAG-NEXT:    global_load_async_to_lds_b32 v2, v[0:1], off offset:4
; SDAG-NEXT:    ; asyncmark
; SDAG-NEXT:    ; asyncmark
; SDAG-NEXT:    global_load_async_to_lds_b32 v3, v[0:1], off offset:4
; SDAG-NEXT:    ; asyncmark
; SDAG-NEXT:    ; wait_asyncmark(1)
; SDAG-NEXT:    s_wait_asynccnt 0x1
; SDAG-NEXT:    ds_load_b32 v6, v2
; SDAG-NEXT:    s_wait_dscnt 0x0
; SDAG-NEXT:    global_store_b32 v[4:5], v6, off
; SDAG-NEXT:    s_set_pc_i64 s[30:31]
;
; GISEL-LABEL: consecutive_asyncmarks_wait1:
; GISEL:       ; %bb.0: ; %entry
; GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
; GISEL-NEXT:    s_wait_kmcnt 0x0
; GISEL-NEXT:    v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GISEL-NEXT:    v_add_nc_u32_e32 v3, 4, v2
; GISEL-NEXT:    s_clause 0x1
; GISEL-NEXT:    global_load_async_to_lds_b32 v2, v[0:1], off offset:4
; GISEL-NEXT:    ; asyncmark
; GISEL-NEXT:    ; asyncmark
; GISEL-NEXT:    global_load_async_to_lds_b32 v3, v[0:1], off offset:4
; GISEL-NEXT:    ; asyncmark
; GISEL-NEXT:    ; wait_asyncmark(1)
; GISEL-NEXT:    s_wait_asynccnt 0x1
; GISEL-NEXT:    ds_load_b32 v4, v2
; GISEL-NEXT:    s_wait_dscnt 0x0
; GISEL-NEXT:    global_store_b32 v[6:7], v4, off
; GISEL-NEXT:    s_set_pc_i64 s[30:31]
entry:
  %lds_gep1 = getelementptr i32, ptr addrspace(3) %lds, i32 1
  call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds, i32 4, i32 0)
  call void @llvm.amdgcn.asyncmark()
  call void @llvm.amdgcn.asyncmark()
  call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %bar, ptr addrspace(3) %lds_gep1, i32 4, i32 0)
  call void @llvm.amdgcn.asyncmark()
  call void @llvm.amdgcn.wait.asyncmark(i16 1)
  %val = load i32, ptr addrspace(3) %lds
  store i32 %val, ptr addrspace(1) %out
  ret void
}