llvm/test/CodeGen/AMDGPU/insertelt-dynamic-chain-widen.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu9.42-amd-amdhsa < %s | FileCheck %s

; A chain of runtime-known index INSERT_VECTOR_ELT on a TypeWidenVector value
; stays in registers when the widened insert is Legal or Promote, instead of
; being combined into one scratch object. Otherwise the chain still uses
; scratch.
;
; On gfx942 <9 x double> widens to <16 x double> and that insert is Promote. A
; per-lane index is lowered with v_cndmask; an index shared by the wave uses
; s_set_gpr_idx.
;
; <3 x half> widens to <4 x half>, whose insert is Custom,
; <5 x half> widens to <8 x half>, whose insert is Expand, and
; <3 x i8> widens to <4 x i8>, which is split, so these still use scratch.

define amdgpu_kernel void @v9f64_divergent_idx(ptr addrspace(1) %out, double %a, double %b) {
; CHECK-LABEL: v9f64_divergent_idx:
; CHECK:       ; %bb.0:
; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
; CHECK-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
; CHECK-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
; CHECK-NEXT:    v_lshlrev_b32_e32 v22, 7, v1
; CHECK-NEXT:    v_and_b32_e32 v0, 7, v0
; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
; CHECK-NEXT:    global_load_dwordx4 v[2:5], v22, s[0:1] offset:32
; CHECK-NEXT:    global_load_dwordx4 v[6:9], v22, s[0:1] offset:48
; CHECK-NEXT:    global_load_dwordx4 v[10:13], v22, s[0:1]
; CHECK-NEXT:    global_load_dwordx4 v[14:17], v22, s[0:1] offset:16
; CHECK-NEXT:    global_load_dwordx4 v[18:21], v22, s[0:1] offset:64
; CHECK-NEXT:    v_mov_b32_e32 v1, s2
; CHECK-NEXT:    s_waitcnt vmcnt(0)
; CHECK-NEXT:    v_mov_b32_e32 v21, s3
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v0
; CHECK-NEXT:    v_add_u32_e32 v20, 1, v0
; CHECK-NEXT:    v_mov_b32_e32 v23, s7
; CHECK-NEXT:    v_mov_b32_e32 v24, s6
; CHECK-NEXT:    v_cndmask_b32_e32 v25, v2, v1, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v26, v3, v21, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v0
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v27, v4, v1, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v28, v5, v21, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v0
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v29, v6, v1, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v30, v7, v21, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v0
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v8, v8, v1, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v9, v9, v21, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v4, v10, v1, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v5, v11, v21, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v6, v12, v1, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v7, v13, v21, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v0
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v10, v14, v1, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v11, v15, v21, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v0
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v2, v16, v1, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v3, v17, v21, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 8, v0
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v0, v18, v1, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v1, v19, v21, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 8, v20
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v17, v1, v23, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v16, v0, v24, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v20
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v23, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v24, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v20
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v1, v11, v23, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v0, v10, v24, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v20
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v7, v7, v23, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v6, v6, v24, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v20
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v5, v5, v23, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v4, v4, v24, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v20
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v11, v9, v23, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v10, v8, v24, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v20
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v9, v30, v23, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v8, v29, v24, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v20
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v15, v28, v23, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v14, v27, v24, vcc
; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v20
; CHECK-NEXT:    s_nop 1
; CHECK-NEXT:    v_cndmask_b32_e32 v13, v26, v23, vcc
; CHECK-NEXT:    v_cndmask_b32_e32 v12, v25, v24, vcc
; CHECK-NEXT:    global_store_dwordx2 v22, v[16:17], s[0:1] offset:64
; CHECK-NEXT:    global_store_dwordx4 v22, v[12:15], s[0:1] offset:32
; CHECK-NEXT:    global_store_dwordx4 v22, v[8:11], s[0:1] offset:48
; CHECK-NEXT:    global_store_dwordx4 v22, v[4:7], s[0:1]
; CHECK-NEXT:    global_store_dwordx4 v22, v[0:3], s[0:1] offset:16
; CHECK-NEXT:    s_endpgm
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr <9 x double>, ptr addrspace(1) %out, i32 %tid
  %vec = load <9 x double>, ptr addrspace(1) %gep
  %i1 = urem i32 %tid, 8
  %i2 = add i32 %i1, 1
  %v1 = insertelement <9 x double> %vec, double %a, i32 %i1
  %v2 = insertelement <9 x double> %v1, double %b, i32 %i2
  store <9 x double> %v2, ptr addrspace(1) %gep
  ret void
}

define amdgpu_kernel void @v9f64_uniform_idx(ptr addrspace(1) %out, double %a, double %b, i32 %idx) {
; CHECK-LABEL: v9f64_uniform_idx:
; CHECK:       ; %bb.0:
; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
; CHECK-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT:    v_lshlrev_b32_e32 v32, 7, v0
; CHECK-NEXT:    s_load_dword s4, s[4:5], 0x18
; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
; CHECK-NEXT:    global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
; CHECK-NEXT:    global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
; CHECK-NEXT:    global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
; CHECK-NEXT:    global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
; CHECK-NEXT:    global_load_dwordx4 v[0:3], v32, s[0:1]
; CHECK-NEXT:    s_waitcnt vmcnt(4)
; CHECK-NEXT:    v_mov_b32_e32 v18, s2
; CHECK-NEXT:    v_mov_b32_e32 v33, s3
; CHECK-NEXT:    v_mov_b32_e32 v34, s6
; CHECK-NEXT:    s_lshl_b32 s4, s4, 1
; CHECK-NEXT:    s_add_i32 s2, s4, 2
; CHECK-NEXT:    s_add_i32 s3, s4, 3
; CHECK-NEXT:    v_mov_b32_e32 v35, s7
; CHECK-NEXT:    s_waitcnt vmcnt(0)
; CHECK-NEXT:    s_set_gpr_idx_on s4, gpr_idx(DST)
; CHECK-NEXT:    v_mov_b32_e32 v0, v18
; CHECK-NEXT:    v_mov_b32_e32 v1, v33
; CHECK-NEXT:    s_set_gpr_idx_off
; CHECK-NEXT:    s_set_gpr_idx_on s2, gpr_idx(DST)
; CHECK-NEXT:    v_mov_b32_e32 v0, v34
; CHECK-NEXT:    s_set_gpr_idx_off
; CHECK-NEXT:    s_set_gpr_idx_on s3, gpr_idx(DST)
; CHECK-NEXT:    v_mov_b32_e32 v0, v35
; CHECK-NEXT:    s_set_gpr_idx_off
; CHECK-NEXT:    global_store_dwordx2 v32, v[16:17], s[0:1] offset:64
; CHECK-NEXT:    global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
; CHECK-NEXT:    global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
; CHECK-NEXT:    global_store_dwordx4 v32, v[0:3], s[0:1]
; CHECK-NEXT:    global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
; CHECK-NEXT:    s_endpgm
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr <9 x double>, ptr addrspace(1) %out, i32 %tid
  %vec = load <9 x double>, ptr addrspace(1) %gep
  %i2 = add i32 %idx, 1
  %v1 = insertelement <9 x double> %vec, double %a, i32 %idx
  %v2 = insertelement <9 x double> %v1, double %b, i32 %i2
  store <9 x double> %v2, ptr addrspace(1) %gep
  ret void
}

define amdgpu_kernel void @v3f16_uniform_idx(ptr addrspace(1) %out, half %a, half %b, i32 %idx) {
; CHECK-LABEL: v3f16_uniform_idx:
; CHECK:       ; %bb.0:
; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
; CHECK-NEXT:    global_load_dwordx2 v[2:3], v0, s[0:1]
; CHECK-NEXT:    s_min_u32 s4, s3, 2
; CHECK-NEXT:    s_add_i32 s3, s3, 1
; CHECK-NEXT:    s_min_u32 s3, s3, 2
; CHECK-NEXT:    v_mov_b32_e32 v1, s2
; CHECK-NEXT:    s_lshl_b32 s2, s4, 1
; CHECK-NEXT:    s_lshl_b32 s3, s3, 1
; CHECK-NEXT:    s_or_b32 s2, s2, 0
; CHECK-NEXT:    s_or_b32 s3, s3, 0
; CHECK-NEXT:    s_waitcnt vmcnt(0)
; CHECK-NEXT:    scratch_store_short off, v3, off offset:4
; CHECK-NEXT:    scratch_store_dword off, v2, off
; CHECK-NEXT:    scratch_store_short off, v1, s2
; CHECK-NEXT:    scratch_store_short_d16_hi off, v1, s3
; CHECK-NEXT:    scratch_load_dwordx2 v[2:3], off, off
; CHECK-NEXT:    s_waitcnt vmcnt(0)
; CHECK-NEXT:    global_store_short v0, v3, s[0:1] offset:4
; CHECK-NEXT:    global_store_dword v0, v2, s[0:1]
; CHECK-NEXT:    s_endpgm
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr <3 x half>, ptr addrspace(1) %out, i32 %tid
  %vec = load <3 x half>, ptr addrspace(1) %gep
  %i2 = add i32 %idx, 1
  %v1 = insertelement <3 x half> %vec, half %a, i32 %idx
  %v2 = insertelement <3 x half> %v1, half %b, i32 %i2
  store <3 x half> %v2, ptr addrspace(1) %gep
  ret void
}

define amdgpu_kernel void @v5f16_uniform_idx(ptr addrspace(1) %out, half %a, half %b, i32 %idx) {
; CHECK-LABEL: v5f16_uniform_idx:
; CHECK:       ; %bb.0:
; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
; CHECK-NEXT:    global_load_dwordx4 v[2:5], v0, s[0:1]
; CHECK-NEXT:    s_min_u32 s4, s3, 4
; CHECK-NEXT:    s_add_i32 s3, s3, 1
; CHECK-NEXT:    s_min_u32 s3, s3, 4
; CHECK-NEXT:    v_mov_b32_e32 v1, s2
; CHECK-NEXT:    s_lshl_b32 s2, s4, 1
; CHECK-NEXT:    s_lshl_b32 s3, s3, 1
; CHECK-NEXT:    s_or_b32 s2, s2, 0
; CHECK-NEXT:    s_or_b32 s3, s3, 0
; CHECK-NEXT:    s_waitcnt vmcnt(0)
; CHECK-NEXT:    scratch_store_short off, v4, off offset:8
; CHECK-NEXT:    scratch_store_dwordx2 off, v[2:3], off
; CHECK-NEXT:    scratch_store_short off, v1, s2
; CHECK-NEXT:    scratch_store_short_d16_hi off, v1, s3
; CHECK-NEXT:    scratch_load_dwordx4 v[2:5], off, off
; CHECK-NEXT:    s_waitcnt vmcnt(0)
; CHECK-NEXT:    global_store_short v0, v4, s[0:1] offset:8
; CHECK-NEXT:    global_store_dwordx2 v0, v[2:3], s[0:1]
; CHECK-NEXT:    s_endpgm
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr <5 x half>, ptr addrspace(1) %out, i32 %tid
  %vec = load <5 x half>, ptr addrspace(1) %gep
  %i2 = add i32 %idx, 1
  %v1 = insertelement <5 x half> %vec, half %a, i32 %idx
  %v2 = insertelement <5 x half> %v1, half %b, i32 %i2
  store <5 x half> %v2, ptr addrspace(1) %gep
  ret void
}

define amdgpu_kernel void @v3i8_uniform_idx(ptr addrspace(1) %out, i8 %a, i8 %b, i32 %idx) {
; CHECK-LABEL: v3i8_uniform_idx:
; CHECK:       ; %bb.0:
; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
; CHECK-NEXT:    global_load_dword v1, v0, s[0:1]
; CHECK-NEXT:    s_min_u32 s4, s3, 2
; CHECK-NEXT:    s_add_i32 s3, s3, 1
; CHECK-NEXT:    s_lshr_b32 s5, s2, 8
; CHECK-NEXT:    s_min_u32 s3, s3, 2
; CHECK-NEXT:    v_mov_b32_e32 v2, s2
; CHECK-NEXT:    s_or_b32 s2, s4, 0
; CHECK-NEXT:    v_mov_b32_e32 v3, s5
; CHECK-NEXT:    s_or_b32 s3, s3, 0
; CHECK-NEXT:    s_waitcnt vmcnt(0)
; CHECK-NEXT:    scratch_store_byte_d16_hi off, v1, off offset:2
; CHECK-NEXT:    scratch_store_short off, v1, off
; CHECK-NEXT:    scratch_store_byte off, v2, s2
; CHECK-NEXT:    scratch_store_byte off, v3, s3
; CHECK-NEXT:    scratch_load_dword v1, off, off
; CHECK-NEXT:    s_waitcnt vmcnt(0)
; CHECK-NEXT:    global_store_byte_d16_hi v0, v1, s[0:1] offset:2
; CHECK-NEXT:    global_store_short v0, v1, s[0:1]
; CHECK-NEXT:    s_endpgm
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr <3 x i8>, ptr addrspace(1) %out, i32 %tid
  %vec = load <3 x i8>, ptr addrspace(1) %gep
  %i2 = add i32 %idx, 1
  %v1 = insertelement <3 x i8> %vec, i8 %a, i32 %idx
  %v2 = insertelement <3 x i8> %v1, i8 %b, i32 %i2
  store <3 x i8> %v2, ptr addrspace(1) %gep
  ret void
}

declare i32 @llvm.amdgcn.workitem.id.x()