test/CodeGen/AMDGPU/cachepolicy.cfs.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu13.10 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX13
; RUN: llc -global-isel=1 -mtriple=amdgpu13.10 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX13

@lds = internal addrspace(3) global [64 x i32] poison

define amdgpu_kernel void @vbuffer_mtbuf_CFS128B(ptr addrspace(1) %out, <4 x i32> %src, i32 %offset, i32 %soffset) {
; GFX13-LABEL: vbuffer_mtbuf_CFS128B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b64 s[6:7], s[4:5], 0x44 nv
; GFX13-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s6
; GFX13-NEXT:    tbuffer_load_format_xyz v[0:2], v0, s[0:3], s7 format:78 offen cfs:CFS_128B
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b96 v3, v[0:2], s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %src, i32 %offset, i32 %soffset, i32 78, i32 128)
  store volatile <3 x float> %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vbuffer_mtbuf_CFS64B(ptr addrspace(1) %out, <4 x i32> %src, i32 %offset, i32 %soffset) {
; GFX13-LABEL: vbuffer_mtbuf_CFS64B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b64 s[6:7], s[4:5], 0x44 nv
; GFX13-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s6
; GFX13-NEXT:    tbuffer_load_format_xyz v[0:2], v0, s[0:3], s7 format:78 offen cfs:CFS_64B
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b96 v3, v[0:2], s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %src, i32 %offset, i32 %soffset, i32 78, i32 256)
  store volatile <3 x float> %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vbuffer_mtbuf_CFS32B(ptr addrspace(1) %out, <4 x i32> %src, i32 %offset, i32 %soffset) {
; GFX13-LABEL: vbuffer_mtbuf_CFS32B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b64 s[6:7], s[4:5], 0x44 nv
; GFX13-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s6
; GFX13-NEXT:    tbuffer_load_format_xyz v[0:2], v0, s[0:3], s7 format:78 offen cfs:CFS_32B
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b96 v3, v[0:2], s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %src, i32 %offset, i32 %soffset, i32 78, i32 384)
  store volatile <3 x float> %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vbuffer_mubuf_CFS128B(ptr addrspace(1) %out, <4 x i32> %src, i32 %offset, i32 %soffset) {
; GFX13-LABEL: vbuffer_mubuf_CFS128B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b64 s[6:7], s[4:5], 0x44 nv
; GFX13-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s6
; GFX13-NEXT:    buffer_load_b32 v0, v0, s[0:3], s7 offen cfs:CFS_128B
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b32 v1, v0, s[0:1]
; GFX13-NEXT:    s_endpgm
  %val = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %src, i32 %offset, i32 %soffset, i32 128)
  store i32 %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vbuffer_mubuf_CFS64B(ptr addrspace(1) %out, <4 x i32> %src, i32 %offset, i32 %soffset) {
; GFX13-LABEL: vbuffer_mubuf_CFS64B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b64 s[6:7], s[4:5], 0x44 nv
; GFX13-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s6
; GFX13-NEXT:    buffer_load_b32 v0, v0, s[0:3], s7 offen cfs:CFS_64B
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b32 v1, v0, s[0:1]
; GFX13-NEXT:    s_endpgm
  %val = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %src, i32 %offset, i32 %soffset, i32 256)
  store i32 %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vbuffer_mubuf_CFS32B(ptr addrspace(1) %out, <4 x i32> %src, i32 %offset, i32 %soffset) {
; GFX13-LABEL: vbuffer_mubuf_CFS32B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b64 s[6:7], s[4:5], 0x44 nv
; GFX13-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s6
; GFX13-NEXT:    buffer_load_b32 v0, v0, s[0:3], s7 offen cfs:CFS_32B
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b32 v1, v0, s[0:1]
; GFX13-NEXT:    s_endpgm
  %val = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %src, i32 %offset, i32 %soffset, i32 384)
  store i32 %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vbuffer_mubuf_format_CFS128B(ptr addrspace(1) %out, <4 x i32> %src, i32 %offset, i32 %soffset) {
; GFX13-LABEL: vbuffer_mubuf_format_CFS128B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b64 s[6:7], s[4:5], 0x44 nv
; GFX13-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s6
; GFX13-NEXT:    buffer_load_format_x v0, v0, s[0:3], s7 offen cfs:CFS_128B
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b32 v1, v0, s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %src, i32 %offset, i32 %soffset, i32 128)
  store volatile float %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vbuffer_mubuf_format_CFS64B(ptr addrspace(1) %out, <4 x i32> %src, i32 %offset, i32 %soffset) {
; GFX13-LABEL: vbuffer_mubuf_format_CFS64B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b64 s[6:7], s[4:5], 0x44 nv
; GFX13-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s6
; GFX13-NEXT:    buffer_load_format_x v0, v0, s[0:3], s7 offen cfs:CFS_64B
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b32 v1, v0, s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %src, i32 %offset, i32 %soffset, i32 256)
  store volatile float %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vbuffer_mubuf_format_CFS32B(ptr addrspace(1) %out, <4 x i32> %src, i32 %offset, i32 %soffset) {
; GFX13-LABEL: vbuffer_mubuf_format_CFS32B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b64 s[6:7], s[4:5], 0x44 nv
; GFX13-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s6
; GFX13-NEXT:    buffer_load_format_x v0, v0, s[0:3], s7 offen cfs:CFS_32B
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b32 v1, v0, s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %src, i32 %offset, i32 %soffset, i32 384)
  store volatile float %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vglobal_CFS128B(ptr addrspace(1) %p) {
; GFX13-LABEL: vglobal_CFS128B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    v_mov_b32_e32 v0, 0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_load_async_to_lds_b32 v0, v0, s[0:1] cfs:CFS_128B
; GFX13-NEXT:    s_endpgm
  call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %p, ptr addrspace(3) @lds, i32 0, i32 128)
  ret void
}

define amdgpu_kernel void @vglobal_CFS64B(ptr addrspace(1) %p) {
; GFX13-LABEL: vglobal_CFS64B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    v_mov_b32_e32 v0, 0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_load_async_to_lds_b32 v0, v0, s[0:1] cfs:CFS_64B
; GFX13-NEXT:    s_endpgm
  call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %p, ptr addrspace(3) @lds, i32 0, i32 256)
  ret void
}

define amdgpu_kernel void @vglobal_CFS32B(ptr addrspace(1) %p) {
; GFX13-LABEL: vglobal_CFS32B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    v_mov_b32_e32 v0, 0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_load_async_to_lds_b32 v0, v0, s[0:1] cfs:CFS_32B
; GFX13-NEXT:    s_endpgm
  call void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %p, ptr addrspace(3) @lds, i32 0, i32 384)
  ret void
}


define amdgpu_kernel void @vimage_CFS128B(ptr addrspace(1) %out, <8 x i32> inreg %src, i32 %s) {
; GFX13-LABEL: vimage_CFS128B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b32 s0, s[4:5], 0x64 nv
; GFX13-NEXT:    s_load_b256 s[8:15], s[4:5], 0x44 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s0
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    image_load v0, v0, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D cfs:CFS_128B
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b32 v1, v0, s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call float @llvm.amdgcn.image.load.1d.f32.i32(i32 1, i32 %s, <8 x i32> %src, i32 0, i32 128)
  store volatile float %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vimage_CFS64B(ptr addrspace(1) %out, <8 x i32> inreg %src, i32 %s) {
; GFX13-LABEL: vimage_CFS64B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b32 s0, s[4:5], 0x64 nv
; GFX13-NEXT:    s_load_b256 s[8:15], s[4:5], 0x44 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s0
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    image_load v0, v0, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D cfs:CFS_64B
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b32 v1, v0, s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call float @llvm.amdgcn.image.load.1d.f32.i32(i32 1, i32 %s, <8 x i32> %src, i32 0, i32 256)
  store volatile float %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vimage_CFS32B(ptr addrspace(1) %out, <8 x i32> inreg %src, i32 %s) {
; GFX13-LABEL: vimage_CFS32B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b32 s0, s[4:5], 0x64 nv
; GFX13-NEXT:    s_load_b256 s[8:15], s[4:5], 0x44 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s0
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    image_load v0, v0, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D cfs:CFS_32B
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b32 v1, v0, s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call float @llvm.amdgcn.image.load.1d.f32.i32(i32 1, i32 %s, <8 x i32> %src, i32 0, i32 384)
  store volatile float %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vsample_CFS128B(ptr addrspace(1) %out, <8 x i32> inreg %src, <4 x i32> inreg %samp, float %s, float %t) {
; GFX13-LABEL: vsample_CFS128B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x2
; GFX13-NEXT:    s_load_b64 s[6:7], s[4:5], 0x74 nv
; GFX13-NEXT:    s_load_b128 s[0:3], s[4:5], 0x64 nv
; GFX13-NEXT:    s_load_b256 s[8:15], s[4:5], 0x44 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v0, s6
; GFX13-NEXT:    v_mov_b32_e32 v1, s7
; GFX13-NEXT:    image_gather4 v[0:3], [v0, v1], s[8:15], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_2D cfs:CFS_128B
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_samplecnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b128 v4, v[0:3], s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 1, float %s, float %t, <8 x i32> %src, <4 x i32> %samp, i1 0, i32 0, i32 128)
  store volatile <4 x float> %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vsample_CFS64B(ptr addrspace(1) %out, <8 x i32> inreg %src, <4 x i32> inreg %samp, float %s, float %t) {
; GFX13-LABEL: vsample_CFS64B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x2
; GFX13-NEXT:    s_load_b64 s[6:7], s[4:5], 0x74 nv
; GFX13-NEXT:    s_load_b128 s[0:3], s[4:5], 0x64 nv
; GFX13-NEXT:    s_load_b256 s[8:15], s[4:5], 0x44 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v0, s6
; GFX13-NEXT:    v_mov_b32_e32 v1, s7
; GFX13-NEXT:    image_gather4 v[0:3], [v0, v1], s[8:15], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_2D cfs:CFS_64B
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_samplecnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b128 v4, v[0:3], s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 1, float %s, float %t, <8 x i32> %src, <4 x i32> %samp, i1 0, i32 0, i32 256)
  store volatile <4 x float> %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vsample_CFS32B(ptr addrspace(1) %out, <8 x i32> inreg %src, <4 x i32> inreg %samp, float %s, float %t) {
; GFX13-LABEL: vsample_CFS32B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    s_clause 0x2
; GFX13-NEXT:    s_load_b64 s[6:7], s[4:5], 0x74 nv
; GFX13-NEXT:    s_load_b128 s[0:3], s[4:5], 0x64 nv
; GFX13-NEXT:    s_load_b256 s[8:15], s[4:5], 0x44 nv
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v0, s6
; GFX13-NEXT:    v_mov_b32_e32 v1, s7
; GFX13-NEXT:    image_gather4 v[0:3], [v0, v1], s[8:15], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_2D cfs:CFS_32B
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_samplecnt 0x0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b128 v4, v[0:3], s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 1, float %s, float %t, <8 x i32> %src, <4 x i32> %samp, i1 0, i32 0, i32 384)
  store volatile <4 x float> %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vbuffer_atomic_rtn_CFS128B(ptr addrspace(1) %out, <4 x i32> %src, i32 %offset, i32 %soffset) {
; GFX13-LABEL: vbuffer_atomic_rtn_CFS128B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13-NEXT:    s_mov_b32 s1, exec_lo
; GFX13-NEXT:    s_mov_b32 s0, exec_lo
; GFX13-NEXT:    ; implicit-def: $vgpr1
; GFX13-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX13-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX13-NEXT:    s_cbranch_execz .LBB18_2
; GFX13-NEXT:  ; %bb.1:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x44 nv
; GFX13-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34 nv
; GFX13-NEXT:    s_bcnt1_i32_b32 s1, s1
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
; GFX13-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], s3 offen th:TH_ATOMIC_RETURN cfs:CFS_128B
; GFX13-NEXT:  .LBB18_2:
; GFX13-NEXT:    s_or_b32 exec_lo, exec_lo, s0
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    v_readfirstlane_b32 s2, v1
; GFX13-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b32 v1, v0, s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 1, <4 x i32> %src, i32 %offset, i32 %soffset, i32 128)
  store volatile i32 %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vbuffer_atomic_rtn_CFS64B(ptr addrspace(1) %out, <4 x i32> %src, i32 %offset, i32 %soffset) {
; GFX13-LABEL: vbuffer_atomic_rtn_CFS64B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13-NEXT:    s_mov_b32 s1, exec_lo
; GFX13-NEXT:    s_mov_b32 s0, exec_lo
; GFX13-NEXT:    ; implicit-def: $vgpr1
; GFX13-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX13-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX13-NEXT:    s_cbranch_execz .LBB19_2
; GFX13-NEXT:  ; %bb.1:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x44 nv
; GFX13-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34 nv
; GFX13-NEXT:    s_bcnt1_i32_b32 s1, s1
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
; GFX13-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], s3 offen th:TH_ATOMIC_RETURN cfs:CFS_64B
; GFX13-NEXT:  .LBB19_2:
; GFX13-NEXT:    s_or_b32 exec_lo, exec_lo, s0
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    v_readfirstlane_b32 s2, v1
; GFX13-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b32 v1, v0, s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 1, <4 x i32> %src, i32 %offset, i32 %soffset, i32 256)
  store volatile i32 %val, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @vbuffer_atomic_rtn_CFS32B(ptr addrspace(1) %out, <4 x i32> %src, i32 %offset, i32 %soffset) {
; GFX13-LABEL: vbuffer_atomic_rtn_CFS32B:
; GFX13:       ; %bb.0:
; GFX13-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13-NEXT:    s_mov_b32 s1, exec_lo
; GFX13-NEXT:    s_mov_b32 s0, exec_lo
; GFX13-NEXT:    ; implicit-def: $vgpr1
; GFX13-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX13-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX13-NEXT:    s_cbranch_execz .LBB20_2
; GFX13-NEXT:  ; %bb.1:
; GFX13-NEXT:    s_clause 0x1
; GFX13-NEXT:    s_load_b64 s[2:3], s[4:5], 0x44 nv
; GFX13-NEXT:    s_load_b128 s[8:11], s[4:5], 0x34 nv
; GFX13-NEXT:    s_bcnt1_i32_b32 s1, s1
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
; GFX13-NEXT:    buffer_atomic_add_u32 v1, v2, s[8:11], s3 offen th:TH_ATOMIC_RETURN cfs:CFS_32B
; GFX13-NEXT:  .LBB20_2:
; GFX13-NEXT:    s_or_b32 exec_lo, exec_lo, s0
; GFX13-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13-NEXT:    s_wait_loadcnt 0x0
; GFX13-NEXT:    v_readfirstlane_b32 s2, v1
; GFX13-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX13-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0
; GFX13-NEXT:    s_wait_kmcnt 0x0
; GFX13-NEXT:    global_store_b32 v1, v0, s[0:1] scope:SCOPE_SYS
; GFX13-NEXT:    s_wait_storecnt 0x0
; GFX13-NEXT:    s_endpgm
  %val = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 1, <4 x i32> %src, i32 %offset, i32 %soffset, i32 384)
  store volatile i32 %val, ptr addrspace(1) %out
  ret void
}