test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -O3 -o - %s | FileCheck %s
; Test for DS pure prefetch pattern: DS loads in loop are NOT used in the same
; iteration but in the next iteration (USE before DEF in program order via phi).
; Expected: "s_wait_dscnt 0" in preheader before loop entry.

define amdgpu_kernel void @ds_prefetch_pattern(ptr addrspace(3) %lds, ptr addrspace(1) %out, i32 %n) {
; CHECK-LABEL: ds_prefetch_pattern:
; CHECK:       ; %bb.0: ; %entry
; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; CHECK-NEXT:    s_clause 0x1
; CHECK-NEXT:    s_load_b32 s1, s[4:5], 0x0
; CHECK-NEXT:    s_load_b32 s0, s[4:5], 0x10
; CHECK-NEXT:    v_and_b32_e32 v12, 0x3ff, v0
; CHECK-NEXT:    s_wait_kmcnt 0x0
; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; CHECK-NEXT:    v_lshl_add_u32 v1, v12, 8, s1
; CHECK-NEXT:    v_mov_b32_e32 v0, 0
; CHECK-NEXT:    s_mov_b32 s1, 0
; CHECK-NEXT:    ds_load_b128 v[4:7], v1
; CHECK-NEXT:    ds_load_b128 v[8:11], v1 offset:16
; CHECK-NEXT:    v_dual_add_nc_u32 v13, 32, v1 :: v_dual_mov_b32 v1, v0
; CHECK-NEXT:    v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, v0
; CHECK-NEXT:    s_wait_dscnt 0x0
; CHECK-NEXT:  .LBB0_1: ; %loop
; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    s_barrier_signal -1
; CHECK-NEXT:    s_wait_dscnt 0x1
; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; CHECK-NEXT:    v_pk_add_f32 v[2:3], v[2:3], v[6:7]
; CHECK-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[4:5]
; CHECK-NEXT:    s_add_co_i32 s1, s1, 1
; CHECK-NEXT:    s_cmp_lt_i32 s1, s0
; CHECK-NEXT:    s_wait_dscnt 0x0
; CHECK-NEXT:    v_pk_add_f32 v[2:3], v[2:3], v[10:11]
; CHECK-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[8:9]
; CHECK-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; CHECK-NEXT:    v_pk_add_f32 v[2:3], v[2:3], v[2:3]
; CHECK-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[0:1]
; CHECK-NEXT:    s_barrier_wait -1
; CHECK-NEXT:    ds_load_b128 v[4:7], v13
; CHECK-NEXT:    ds_load_b128 v[8:11], v13 offset:16
; CHECK-NEXT:    v_add_nc_u32_e32 v13, 32, v13
; CHECK-NEXT:    s_cbranch_scc1 .LBB0_1
; CHECK-NEXT:  ; %bb.2: ; %exit
; CHECK-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
; CHECK-NEXT:    s_wait_kmcnt 0x0
; CHECK-NEXT:    global_store_b128 v12, v[0:3], s[0:1] scale_offset
; CHECK-NEXT:    s_endpgm
entry:
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  %base = shl i32 %tid, 4

  ; Initial prefetch loads in preheader
  %ptr.init = getelementptr <4 x float>, ptr addrspace(3) %lds, i32 %base
  %init.v1 = load <4 x float>, ptr addrspace(3) %ptr.init, align 16
  %ptr.init2 = getelementptr <4 x float>, ptr addrspace(3) %lds, i32 %base, i32 4
  %init.v2 = load <4 x float>, ptr addrspace(3) %ptr.init2, align 16

  br label %loop

loop:
  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
  %acc = phi <4 x float> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
  ; These hold values loaded in previous iteration (prefetch pattern)
  %prefetch1 = phi <4 x float> [ %init.v1, %entry ], [ %next.v1, %loop ]
  %prefetch2 = phi <4 x float> [ %init.v2, %entry ], [ %next.v2, %loop ]

  ; Use prefetched values from previous iteration
  %use1 = fadd <4 x float> %acc, %prefetch1
  %use2 = fadd <4 x float> %use1, %prefetch2

  ; Barrier
  call void @llvm.amdgcn.s.barrier()

  ; Compute next iteration's address
  %next.i = add i32 %i, 1
  %next.off = mul i32 %next.i, 2
  %next.base = add i32 %base, %next.off

  ; Prefetch loads for NEXT iteration (after barrier)
  ; These are NOT used in the same iteration - pure prefetch pattern
  %ptr.next1 = getelementptr <4 x float>, ptr addrspace(3) %lds, i32 %next.base
  %next.v1 = load <4 x float>, ptr addrspace(3) %ptr.next1, align 16

  %ptr.next2 = getelementptr <4 x float>, ptr addrspace(3) %lds, i32 %next.base, i32 4
  %next.v2 = load <4 x float>, ptr addrspace(3) %ptr.next2, align 16

  %acc.next = fadd <4 x float> %use2, %use2

  %i.next = add i32 %i, 1
  %cond = icmp slt i32 %i.next, %n
  br i1 %cond, label %loop, label %exit

exit:
  %out.ptr = getelementptr <4 x float>, ptr addrspace(1) %out, i32 %tid
  store <4 x float> %acc.next, ptr addrspace(1) %out.ptr, align 16
  ret void
}

declare i32 @llvm.amdgcn.workitem.id.x()
declare void @llvm.amdgcn.s.barrier()