llvm/test/CodeGen/AMDGPU/inline-asm-vgpr-sgpr-copy.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck %s

; This is a regression test for a bug where SIFixSGPRCopies would incorrectly
; keep the value as VGPR when it should be converted to SGPR for inline asm.

define <4 x float> @test_sgpr_constraint_bug(ptr addrspace(5) %buf_desc_ptr) {
; CHECK-LABEL: test_sgpr_constraint_bug:
; CHECK:       ; %bb.0:
; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT:    scratch_load_dwordx4 v[4:7], v0, off
; CHECK-NEXT:    v_mov_b32_e32 v0, 0
; CHECK-NEXT:    v_mov_b32_e32 v1, v0
; CHECK-NEXT:    v_mov_b32_e32 v2, v0
; CHECK-NEXT:    v_mov_b32_e32 v3, v0
; CHECK-NEXT:    s_mov_b64 s[4:5], exec
; CHECK-NEXT:    v_mov_b32_e32 v8, 1
; CHECK-NEXT:    s_waitcnt vmcnt(0)
; CHECK-NEXT:    v_readfirstlane_b32 s3, v7
; CHECK-NEXT:    v_readfirstlane_b32 s2, v6
; CHECK-NEXT:    v_readfirstlane_b32 s1, v5
; CHECK-NEXT:    v_readfirstlane_b32 s0, v4
; CHECK-NEXT:    ;;#ASMSTART
; CHECK-NEXT:    v_cmpx_le_u32 exec, 1, v8
; CHECK-NEXT:  buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:0
; CHECK-NEXT:  s_mov_b64 exec s[4:5]
; CHECK-NEXT:    ;;#ASMEND
; CHECK-NEXT:    s_setpc_b64 s[30:31]
  %rsrc = load <4 x i32>, ptr addrspace(5) %buf_desc_ptr, align 16

  %exec = call i64 @llvm.amdgcn.ballot.i64(i1 true)

  %result = call <4 x float> asm sideeffect
    "v_cmpx_le_u32 exec, 1, $4\0Abuffer_load_dwordx4 $0, $1, $2, 0 offen offset:$3\0As_mov_b64 exec $5",
    "=v,v,s,n,v,s,0,~{memory}"
    (i32 0, <4 x i32> %rsrc, i32 0, i32 1, i64 %exec, <4 x float> zeroinitializer)

  ret <4 x float> %result
}

;inline asm does not construct waterfall; user should be responsible
define amdgpu_kernel void @inlineasm_sgpr_constraint_divergent_value(ptr addrspace(1) %out, i32 %uniform) {
; CHECK-LABEL: inlineasm_sgpr_constraint_divergent_value:
; CHECK:       ; %bb.0: ; %entry
; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT:    s_nop 0
; CHECK-NEXT:    v_readfirstlane_b32 s2, v0
; CHECK-NEXT:    ;;#ASMSTART
; CHECK-NEXT:    s_add_u32 s2, s2, 1
; CHECK-NEXT:    ;;#ASMEND
; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; CHECK-NEXT:    s_nop 0
; CHECK-NEXT:    v_mov_b32_e32 v1, s2
; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
; CHECK-NEXT:    global_store_dword v0, v1, s[0:1]
; CHECK-NEXT:    s_endpgm
entry:
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  %result = call i32 asm sideeffect "s_add_u32 $0, $1, 1", "=s,s"(i32 %tid)
  %gep = getelementptr i32, ptr addrspace(1) %out, i32 %tid
  store i32 %result, ptr addrspace(1) %gep
  ret void
}

define amdgpu_kernel void @inlineasm_with_salu_to_valu_uses(ptr addrspace(1) %out, i32 %base) {
; CHECK-LABEL: inlineasm_with_salu_to_valu_uses:
; CHECK:       ; %bb.0: ; %entry
; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT:    v_add_u32_e32 v1, 10, v0
; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; CHECK-NEXT:    v_readfirstlane_b32 s2, v1
; CHECK-NEXT:    ;;#ASMSTART
; CHECK-NEXT:    s_add_u32 s2, s2, 5
; CHECK-NEXT:    ;;#ASMEND
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_mad_u32_u24 v1, v1, 3, s2
; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
; CHECK-NEXT:    global_store_dword v0, v1, s[0:1]
; CHECK-NEXT:    s_endpgm
entry:
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  %derived = add i32 %tid, 10

  %asm_result = call i32 asm sideeffect "s_add_u32 $0, $1, 5", "=s,s"(i32 %derived)

  %mul_result = mul i32 %derived, 3
  %combined = add i32 %asm_result, %mul_result
  %gep = getelementptr i32, ptr addrspace(1) %out, i32 %tid
  store i32 %combined, ptr addrspace(1) %gep
  ret void
}

; Test case with divergent value used in inline asm AND feeding into
; a waterfall loop
define amdgpu_kernel void @inlineasm_and_waterfall_same_value(ptr addrspace(1) %out, ptr addrspace(1) %descriptors) {
; CHECK-LABEL: inlineasm_and_waterfall_same_value:
; CHECK:       ; %bb.0: ; %entry
; CHECK-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x24
; CHECK-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 4, v4
; CHECK-NEXT:    v_readfirstlane_b32 s0, v4
; CHECK-NEXT:    ;;#ASMSTART
; CHECK-NEXT:    s_add_u32 s12, s0, 100
; CHECK-NEXT:    ;;#ASMEND
; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
; CHECK-NEXT:    global_load_dwordx4 v[0:3], v0, s[10:11]
; CHECK-NEXT:    s_mov_b64 s[2:3], exec
; CHECK-NEXT:    s_waitcnt vmcnt(0)
; CHECK-NEXT:  .LBB3_1: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    v_readfirstlane_b32 s4, v0
; CHECK-NEXT:    v_readfirstlane_b32 s5, v1
; CHECK-NEXT:    v_readfirstlane_b32 s6, v2
; CHECK-NEXT:    v_readfirstlane_b32 s7, v3
; CHECK-NEXT:    v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]
; CHECK-NEXT:    s_nop 0
; CHECK-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; CHECK-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
; CHECK-NEXT:    s_and_saveexec_b64 s[0:1], s[0:1]
; CHECK-NEXT:    buffer_load_dword v5, v4, s[4:7], 0 offen
; CHECK-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT:    s_xor_b64 exec, exec, s[0:1]
; CHECK-NEXT:    s_cbranch_execnz .LBB3_1
; CHECK-NEXT:  ; %bb.2:
; CHECK-NEXT:    s_mov_b64 exec, s[2:3]
; CHECK-NEXT:    s_waitcnt vmcnt(0)
; CHECK-NEXT:    v_add_u32_e32 v0, s12, v5
; CHECK-NEXT:    v_lshlrev_b32_e32 v1, 2, v4
; CHECK-NEXT:    global_store_dword v1, v0, s[8:9]
; CHECK-NEXT:    s_endpgm
entry:
  %tid = call i32 @llvm.amdgcn.workitem.id.x()

  %asm_result = call i32 asm sideeffect "s_add_u32 $0, $1, 100", "=s,s"(i32 %tid)

  %desc_ptr = getelementptr <4 x i32>, ptr addrspace(1) %descriptors, i32 %tid
  %desc = load <4 x i32>, ptr addrspace(1) %desc_ptr
  %buffer_result = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %desc, i32 %tid, i32 0, i32 0)

  %combined = add i32 %asm_result, %buffer_result
  %out_ptr = getelementptr i32, ptr addrspace(1) %out, i32 %tid
  store i32 %combined, ptr addrspace(1) %out_ptr
  ret void
}

; Test case with multiple divergent SGPR inputs to verify all are handled
define amdgpu_kernel void @multiple_divergent_sgpr_inputs(ptr addrspace(1) %out) {
; CHECK-LABEL: multiple_divergent_sgpr_inputs:
; CHECK:       ; %bb.0: ; %entry
; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT:    v_add_u32_e32 v1, 10, v0
; CHECK-NEXT:    v_add_u32_e32 v2, 20, v0
; CHECK-NEXT:    v_readfirstlane_b32 s2, v0
; CHECK-NEXT:    v_readfirstlane_b32 s3, v1
; CHECK-NEXT:    v_readfirstlane_b32 s4, v2
; CHECK-NEXT:    ;;#ASMSTART
; CHECK-NEXT:    s_add_u32 s2, s2, s3
; CHECK-NEXT:  s_add_u32 s2, s2, s4
; CHECK-NEXT:    ;;#ASMEND
; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; CHECK-NEXT:    v_mov_b32_e32 v1, s2
; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
; CHECK-NEXT:    global_store_dword v0, v1, s[0:1]
; CHECK-NEXT:    s_endpgm
entry:
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  %val1 = add i32 %tid, 10
  %val2 = add i32 %tid, 20
  ; Three separate divergent SGPR inputs
  %result = call i32 asm sideeffect "s_add_u32 $0, $1, $2\0As_add_u32 $0, $0, $3", "=s,s,s,s"(i32 %tid, i32 %val1, i32 %val2)
  %gep = getelementptr i32, ptr addrspace(1) %out, i32 %tid
  store i32 %result, ptr addrspace(1) %gep
  ret void
}

define amdgpu_kernel void @same_value_repeated_sgpr_inputs(ptr addrspace(1) %out) {
; CHECK-LABEL: same_value_repeated_sgpr_inputs:
; CHECK:       ; %bb.0: ; %entry
; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT:    s_nop 0
; CHECK-NEXT:    v_readfirstlane_b32 s2, v0
; CHECK-NEXT:    ;;#ASMSTART
; CHECK-NEXT:    s_add_u32 s3, s2, s2
; CHECK-NEXT:  s_add_u32 s3, s3, s2
; CHECK-NEXT:    ;;#ASMEND
; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; CHECK-NEXT:    v_mov_b32_e32 v1, s3
; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
; CHECK-NEXT:    global_store_dword v0, v1, s[0:1]
; CHECK-NEXT:    s_endpgm
entry:
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  ; Same value %tid used three times as SGPR input ($1, $2, $3).
  ; Early-clobber '&' ensures $0 gets a different register from the inputs.
  %result = call i32 asm sideeffect "s_add_u32 $0, $1, $2\0As_add_u32 $0, $0, $3", "=&s,s,s,s"(i32 %tid, i32 %tid, i32 %tid)
  %gep = getelementptr i32, ptr addrspace(1) %out, i32 %tid
  store i32 %result, ptr addrspace(1) %gep
  ret void
}

define amdgpu_kernel void @physreg_sgpr_constraint_divergent_value(ptr addrspace(1) %out) {
; CHECK-LABEL: physreg_sgpr_constraint_divergent_value:
; CHECK:       ; %bb.0: ; %entry
; CHECK-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT:    s_nop 0
; CHECK-NEXT:    v_readfirstlane_b32 s0, v0
; CHECK-NEXT:    ;;#ASMSTART
; CHECK-NEXT:    s_add_u32 s0, s0, 1
; CHECK-NEXT:    ;;#ASMEND
; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; CHECK-NEXT:    s_nop 0
; CHECK-NEXT:    v_mov_b32_e32 v1, s0
; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
; CHECK-NEXT:    global_store_dword v0, v1, s[2:3]
; CHECK-NEXT:    s_endpgm
entry:
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  ; Use physical register constraint {s0} with divergent value
  %result = call i32 asm sideeffect "s_add_u32 $0, $0, 1", "={s0},{s0}"(i32 %tid)
  %gep = getelementptr i32, ptr addrspace(1) %out, i32 %tid
  store i32 %result, ptr addrspace(1) %gep
  ret void
}

define amdgpu_kernel void @sgpr_tuple_i64_constraint(ptr addrspace(1) %out) {
; CHECK-LABEL: sgpr_tuple_i64_constraint:
; CHECK:       ; %bb.0: ; %entry
; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT:    s_mov_b32 s3, 0
; CHECK-NEXT:    v_readfirstlane_b32 s2, v0
; CHECK-NEXT:    ;;#ASMSTART
; CHECK-NEXT:    s_nop 0
; CHECK-NEXT:    ;;#ASMEND
; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; CHECK-NEXT:    s_nop 0
; CHECK-NEXT:    v_mov_b32_e32 v1, s2
; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
; CHECK-NEXT:    global_store_dword v0, v1, s[0:1]
; CHECK-NEXT:    s_endpgm
entry:
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  %tid64 = zext i32 %tid to i64
  %result = call i64 asm sideeffect "s_nop 0", "=s,s"(i64 %tid64)
  %trunc = trunc i64 %result to i32
  %gep = getelementptr i32, ptr addrspace(1) %out, i32 %tid
  store i32 %trunc, ptr addrspace(1) %gep
  ret void
}

define amdgpu_kernel void @sgpr_tuple_v2i32_constraint(ptr addrspace(1) %out, ptr addrspace(1) %in) {
; CHECK-LABEL: sgpr_tuple_v2i32_constraint:
; CHECK:       ; %bb.0: ; %entry
; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; CHECK-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 3, v2
; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
; CHECK-NEXT:    global_load_dwordx2 v[0:1], v0, s[2:3]
; CHECK-NEXT:    s_waitcnt vmcnt(0)
; CHECK-NEXT:    v_readfirstlane_b32 s3, v1
; CHECK-NEXT:    v_readfirstlane_b32 s2, v0
; CHECK-NEXT:    ;;#ASMSTART
; CHECK-NEXT:    s_nop 0
; CHECK-NEXT:    ;;#ASMEND
; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v2
; CHECK-NEXT:    s_nop 0
; CHECK-NEXT:    v_mov_b32_e32 v1, s2
; CHECK-NEXT:    global_store_dword v0, v1, s[0:1]
; CHECK-NEXT:    s_endpgm
entry:
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  %gep_in = getelementptr <2 x i32>, ptr addrspace(1) %in, i32 %tid
  %val = load <2 x i32>, ptr addrspace(1) %gep_in
  %result = call <2 x i32> asm sideeffect "s_nop 0", "=s,s"(<2 x i32> %val)
  %elem = extractelement <2 x i32> %result, i32 0
  %gep = getelementptr i32, ptr addrspace(1) %out, i32 %tid
  store i32 %elem, ptr addrspace(1) %gep
  ret void
}