; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu6.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX6
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX7
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX8
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX9
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX10
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa -amdgpu-enable-vopd=0 -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX11
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX1200-SDAG
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00-amd-amdhsa -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX1200-GISEL
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-wait-xcnt -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX1250-SDAG
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-wait-xcnt -verify-machineinstrs < %s | FileCheck %s --check-prefix=GFX1250-GISEL

define amdgpu_kernel void @ptr_s_buffer_load_i32(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc, i32 inreg %offset) {
;
; GFX6-LABEL: ptr_s_buffer_load_i32:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
; GFX6-NEXT:    s_load_dword s4, s[8:9], 0x8
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    s_buffer_load_dword s4, s[0:3], s4
; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
; GFX6-NEXT:    s_mov_b32 s2, -1
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_mov_b32_e32 v0, s4
; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: ptr_s_buffer_load_i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x8
; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX7-NEXT:    s_add_i32 s12, s12, s17
; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    s_buffer_load_dword s0, s[0:3], s6
; GFX7-NEXT:    v_mov_b32_e32 v0, s4
; GFX7-NEXT:    v_mov_b32_e32 v1, s5
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    v_mov_b32_e32 v2, s0
; GFX7-NEXT:    flat_store_dword v[0:1], v2
; GFX7-NEXT:    s_endpgm
;
; GFX8-LABEL: ptr_s_buffer_load_i32:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dword s6, s[8:9], 0x20
; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX8-NEXT:    s_add_i32 s12, s12, s17
; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    s_buffer_load_dword s0, s[0:3], s6
; GFX8-NEXT:    v_mov_b32_e32 v0, s4
; GFX8-NEXT:    v_mov_b32_e32 v1, s5
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v2, s0
; GFX8-NEXT:    flat_store_dword v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: ptr_s_buffer_load_i32:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX9-NEXT:    s_load_dword s6, s[8:9], 0x20
; GFX9-NEXT:    v_mov_b32_e32 v0, 0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_buffer_load_dword s7, s[0:3], s6 offset:0x0
; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_mov_b32_e32 v1, s7
; GFX9-NEXT:    global_store_dword v0, v1, s[4:5]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: ptr_s_buffer_load_i32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_clause 0x1
; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX10-NEXT:    s_load_dword s6, s[8:9], 0x20
; GFX10-NEXT:    v_mov_b32_e32 v0, 0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_buffer_load_dword s7, s[0:3], s6 offset:0x0
; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v1, s7
; GFX10-NEXT:    global_store_dword v0, v1, s[4:5]
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: ptr_s_buffer_load_i32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_clause 0x1
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x20
; GFX11-NEXT:    v_mov_b32_e32 v0, 0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_buffer_load_b32 s2, s[0:3], s6 offset:0x0
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_mov_b32_e32 v1, s2
; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT:    s_endpgm
;
; GFX1200-SDAG-LABEL: ptr_s_buffer_load_i32:
; GFX1200-SDAG:       ; %bb.0:
; GFX1200-SDAG-NEXT:    s_clause 0x1
; GFX1200-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX1200-SDAG-NEXT:    s_load_b32 s6, s[4:5], 0x20
; GFX1200-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT:    s_buffer_load_b32 s2, s[0:3], s6 offset:0x0
; GFX1200-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
; GFX1200-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1200-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX1200-SDAG-NEXT:    s_endpgm
;
; GFX1200-GISEL-LABEL: ptr_s_buffer_load_i32:
; GFX1200-GISEL:       ; %bb.0:
; GFX1200-GISEL-NEXT:    s_clause 0x2
; GFX1200-GISEL-NEXT:    s_load_b32 s6, s[4:5], 0x20
; GFX1200-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX1200-GISEL-NEXT:    s_load_b64 s[4:5], s[4:5], 0x0
; GFX1200-GISEL-NEXT:    v_mov_b32_e32 v1, 0
; GFX1200-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1200-GISEL-NEXT:    s_buffer_load_b32 s0, s[0:3], s6 offset:0x0
; GFX1200-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1200-GISEL-NEXT:    v_mov_b32_e32 v0, s0
; GFX1200-GISEL-NEXT:    global_store_b32 v1, v0, s[4:5]
; GFX1200-GISEL-NEXT:    s_endpgm
;
; GFX1250-SDAG-LABEL: ptr_s_buffer_load_i32:
; GFX1250-SDAG:       ; %bb.0:
; GFX1250-SDAG-NEXT:    global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-SDAG-NEXT:    v_nop
; GFX1250-SDAG-NEXT:    s_clause 0x1
; GFX1250-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10 nv
; GFX1250-SDAG-NEXT:    s_load_b32 s8, s[4:5], 0x20 nv
; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT:    s_buffer_load_b32 s9, s[0:3], s8 offset:0x0 nv
; GFX1250-SDAG-NEXT:    s_load_b64 s[6:7], s[4:5], 0x0 nv
; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s9
; GFX1250-SDAG-NEXT:    global_store_b32 v0, v1, s[6:7]
; GFX1250-SDAG-NEXT:    s_endpgm
;
; GFX1250-GISEL-LABEL: ptr_s_buffer_load_i32:
; GFX1250-GISEL:       ; %bb.0:
; GFX1250-GISEL-NEXT:    global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-GISEL-NEXT:    v_nop
; GFX1250-GISEL-NEXT:    s_clause 0x2
; GFX1250-GISEL-NEXT:    s_load_b32 s8, s[4:5], 0x20 nv
; GFX1250-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10 nv
; GFX1250-GISEL-NEXT:    s_load_b64 s[6:7], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT:    v_mov_b32_e32 v1, 0
; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT:    s_buffer_load_b32 s0, s[0:3], s8 offset:0x0 nv
; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT:    v_mov_b32_e32 v0, s0
; GFX1250-GISEL-NEXT:    global_store_b32 v1, v0, s[6:7]
; GFX1250-GISEL-NEXT:    s_endpgm
  %load = call i32 @llvm.amdgcn.ptr.s.buffer.load.i32(ptr addrspace(8) %rsrc, i32 %offset, i32 0), !invariant.load !0
  store i32 %load, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @ptr_s_buffer_load_v2i32(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc, i32 inreg %offset) {
;
; GFX6-LABEL: ptr_s_buffer_load_v2i32:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
; GFX6-NEXT:    s_load_dword s4, s[8:9], 0x8
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    s_buffer_load_dwordx2 s[4:5], s[0:3], s4
; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
; GFX6-NEXT:    s_mov_b32 s2, -1
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_mov_b32_e32 v0, s4
; GFX6-NEXT:    v_mov_b32_e32 v1, s5
; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: ptr_s_buffer_load_v2i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x8
; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX7-NEXT:    s_add_i32 s12, s12, s17
; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    s_buffer_load_dwordx2 s[0:1], s[0:3], s6
; GFX7-NEXT:    v_mov_b32_e32 v0, s4
; GFX7-NEXT:    v_mov_b32_e32 v1, s5
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    v_mov_b32_e32 v3, s1
; GFX7-NEXT:    v_mov_b32_e32 v2, s0
; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT:    s_endpgm
;
; GFX8-LABEL: ptr_s_buffer_load_v2i32:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dword s6, s[8:9], 0x20
; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX8-NEXT:    s_add_i32 s12, s12, s17
; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    s_buffer_load_dwordx2 s[0:1], s[0:3], s6
; GFX8-NEXT:    v_mov_b32_e32 v0, s4
; GFX8-NEXT:    v_mov_b32_e32 v1, s5
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v3, s1
; GFX8-NEXT:    v_mov_b32_e32 v2, s0
; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: ptr_s_buffer_load_v2i32:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX9-NEXT:    s_load_dword s10, s[8:9], 0x20
; GFX9-NEXT:    v_mov_b32_e32 v2, 0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_buffer_load_dwordx2 s[4:5], s[0:3], s10 offset:0x0
; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_mov_b32_e32 v0, s4
; GFX9-NEXT:    v_mov_b32_e32 v1, s5
; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: ptr_s_buffer_load_v2i32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_clause 0x1
; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX10-NEXT:    s_load_dword s10, s[8:9], 0x20
; GFX10-NEXT:    v_mov_b32_e32 v2, 0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_buffer_load_dwordx2 s[4:5], s[0:3], s10 offset:0x0
; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[8:9], 0x0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v0, s4
; GFX10-NEXT:    v_mov_b32_e32 v1, s5
; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: ptr_s_buffer_load_v2i32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_clause 0x1
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x20
; GFX11-NEXT:    v_mov_b32_e32 v2, 0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_buffer_load_b64 s[0:1], s[0:3], s6 offset:0x0
; GFX11-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_mov_b32_e32 v0, s0
; GFX11-NEXT:    v_mov_b32_e32 v1, s1
; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
; GFX11-NEXT:    s_endpgm
;
; GFX1200-SDAG-LABEL: ptr_s_buffer_load_v2i32:
; GFX1200-SDAG:       ; %bb.0:
; GFX1200-SDAG-NEXT:    s_clause 0x1
; GFX1200-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX1200-SDAG-NEXT:    s_load_b32 s6, s[4:5], 0x20
; GFX1200-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT:    s_buffer_load_b64 s[0:1], s[0:3], s6 offset:0x0
; GFX1200-SDAG-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
; GFX1200-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
; GFX1200-SDAG-NEXT:    v_mov_b32_e32 v0, s0
; GFX1200-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
; GFX1200-SDAG-NEXT:    s_endpgm
;
; GFX1200-GISEL-LABEL: ptr_s_buffer_load_v2i32:
; GFX1200-GISEL:       ; %bb.0:
; GFX1200-GISEL-NEXT:    s_clause 0x2
; GFX1200-GISEL-NEXT:    s_load_b32 s6, s[4:5], 0x20
; GFX1200-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX1200-GISEL-NEXT:    s_load_b64 s[4:5], s[4:5], 0x0
; GFX1200-GISEL-NEXT:    v_mov_b32_e32 v2, 0
; GFX1200-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1200-GISEL-NEXT:    s_buffer_load_b64 s[0:1], s[0:3], s6 offset:0x0
; GFX1200-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1200-GISEL-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX1200-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
; GFX1200-GISEL-NEXT:    s_endpgm
;
; GFX1250-SDAG-LABEL: ptr_s_buffer_load_v2i32:
; GFX1250-SDAG:       ; %bb.0:
; GFX1250-SDAG-NEXT:    global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-SDAG-NEXT:    v_nop
; GFX1250-SDAG-NEXT:    s_clause 0x1
; GFX1250-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10 nv
; GFX1250-SDAG-NEXT:    s_load_b32 s10, s[4:5], 0x20 nv
; GFX1250-SDAG-NEXT:    v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT:    s_buffer_load_b64 s[6:7], s[0:3], s10 offset:0x0 nv
; GFX1250-SDAG-NEXT:    s_load_b64 s[8:9], s[4:5], 0x0 nv
; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]
; GFX1250-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[8:9]
; GFX1250-SDAG-NEXT:    s_endpgm
;
; GFX1250-GISEL-LABEL: ptr_s_buffer_load_v2i32:
; GFX1250-GISEL:       ; %bb.0:
; GFX1250-GISEL-NEXT:    global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-GISEL-NEXT:    v_nop
; GFX1250-GISEL-NEXT:    s_clause 0x2
; GFX1250-GISEL-NEXT:    s_load_b32 s8, s[4:5], 0x20 nv
; GFX1250-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10 nv
; GFX1250-GISEL-NEXT:    s_load_b64 s[6:7], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT:    v_mov_b32_e32 v2, 0
; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT:    s_buffer_load_b64 s[0:1], s[0:3], s8 offset:0x0 nv
; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[6:7]
; GFX1250-GISEL-NEXT:    s_endpgm
  %load = call <2 x i32> @llvm.amdgcn.ptr.s.buffer.load.v2i32(ptr addrspace(8) %rsrc, i32 %offset, i32 0), !invariant.load !0
  store <2 x i32> %load, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @ptr_s_buffer_load_v4i32(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc, i32 inreg %offset) {
;
; GFX6-LABEL: ptr_s_buffer_load_v4i32:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
; GFX6-NEXT:    s_load_dword s4, s[8:9], 0x8
; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
; GFX6-NEXT:    s_mov_b32 s6, -1
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    s_buffer_load_dwordx4 s[0:3], s[0:3], s4
; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_mov_b32_e32 v0, s0
; GFX6-NEXT:    v_mov_b32_e32 v1, s1
; GFX6-NEXT:    v_mov_b32_e32 v2, s2
; GFX6-NEXT:    v_mov_b32_e32 v3, s3
; GFX6-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: ptr_s_buffer_load_v4i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_load_dword s6, s[8:9], 0x8
; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX7-NEXT:    s_add_i32 s12, s12, s17
; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    s_buffer_load_dwordx4 s[0:3], s[0:3], s6
; GFX7-NEXT:    v_mov_b32_e32 v4, s4
; GFX7-NEXT:    v_mov_b32_e32 v5, s5
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    v_mov_b32_e32 v0, s0
; GFX7-NEXT:    v_mov_b32_e32 v1, s1
; GFX7-NEXT:    v_mov_b32_e32 v2, s2
; GFX7-NEXT:    v_mov_b32_e32 v3, s3
; GFX7-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-NEXT:    s_endpgm
;
; GFX8-LABEL: ptr_s_buffer_load_v4i32:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dword s6, s[8:9], 0x20
; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX8-NEXT:    s_add_i32 s12, s12, s17
; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    s_buffer_load_dwordx4 s[0:3], s[0:3], s6
; GFX8-NEXT:    v_mov_b32_e32 v4, s4
; GFX8-NEXT:    v_mov_b32_e32 v5, s5
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v0, s0
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_mov_b32_e32 v2, s2
; GFX8-NEXT:    v_mov_b32_e32 v3, s3
; GFX8-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: ptr_s_buffer_load_v4i32:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX9-NEXT:    s_load_dword s12, s[8:9], 0x20
; GFX9-NEXT:    v_mov_b32_e32 v4, 0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_buffer_load_dwordx4 s[4:7], s[0:3], s12 offset:0x0
; GFX9-NEXT:    s_load_dwordx2 s[10:11], s[8:9], 0x0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_mov_b32_e32 v0, s4
; GFX9-NEXT:    v_mov_b32_e32 v1, s5
; GFX9-NEXT:    v_mov_b32_e32 v2, s6
; GFX9-NEXT:    v_mov_b32_e32 v3, s7
; GFX9-NEXT:    global_store_dwordx4 v4, v[0:3], s[10:11]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: ptr_s_buffer_load_v4i32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_clause 0x1
; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX10-NEXT:    s_load_dword s12, s[8:9], 0x20
; GFX10-NEXT:    v_mov_b32_e32 v4, 0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_buffer_load_dwordx4 s[4:7], s[0:3], s12 offset:0x0
; GFX10-NEXT:    s_load_dwordx2 s[10:11], s[8:9], 0x0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v0, s4
; GFX10-NEXT:    v_mov_b32_e32 v1, s5
; GFX10-NEXT:    v_mov_b32_e32 v2, s6
; GFX10-NEXT:    v_mov_b32_e32 v3, s7
; GFX10-NEXT:    global_store_dwordx4 v4, v[0:3], s[10:11]
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: ptr_s_buffer_load_v4i32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_clause 0x1
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x20
; GFX11-NEXT:    v_mov_b32_e32 v4, 0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_buffer_load_b128 s[0:3], s[0:3], s6 offset:0x0
; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_mov_b32_e32 v0, s0
; GFX11-NEXT:    v_mov_b32_e32 v1, s1
; GFX11-NEXT:    v_mov_b32_e32 v2, s2
; GFX11-NEXT:    v_mov_b32_e32 v3, s3
; GFX11-NEXT:    global_store_b128 v4, v[0:3], s[4:5]
; GFX11-NEXT:    s_endpgm
;
; GFX1200-SDAG-LABEL: ptr_s_buffer_load_v4i32:
; GFX1200-SDAG:       ; %bb.0:
; GFX1200-SDAG-NEXT:    s_clause 0x1
; GFX1200-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX1200-SDAG-NEXT:    s_load_b32 s6, s[4:5], 0x20
; GFX1200-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT:    s_buffer_load_b128 s[0:3], s[0:3], s6 offset:0x0
; GFX1200-SDAG-NEXT:    s_load_b64 s[4:5], s[4:5], 0x0
; GFX1200-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s1
; GFX1200-SDAG-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
; GFX1200-SDAG-NEXT:    v_mov_b32_e32 v2, s2
; GFX1200-SDAG-NEXT:    global_store_b128 v4, v[0:3], s[4:5]
; GFX1200-SDAG-NEXT:    s_endpgm
;
; GFX1200-GISEL-LABEL: ptr_s_buffer_load_v4i32:
; GFX1200-GISEL:       ; %bb.0:
; GFX1200-GISEL-NEXT:    s_clause 0x2
; GFX1200-GISEL-NEXT:    s_load_b32 s6, s[4:5], 0x20
; GFX1200-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX1200-GISEL-NEXT:    s_load_b64 s[4:5], s[4:5], 0x0
; GFX1200-GISEL-NEXT:    v_mov_b32_e32 v4, 0
; GFX1200-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1200-GISEL-NEXT:    s_buffer_load_b128 s[0:3], s[0:3], s6 offset:0x0
; GFX1200-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1200-GISEL-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX1200-GISEL-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX1200-GISEL-NEXT:    global_store_b128 v4, v[0:3], s[4:5]
; GFX1200-GISEL-NEXT:    s_endpgm
;
; GFX1250-SDAG-LABEL: ptr_s_buffer_load_v4i32:
; GFX1250-SDAG:       ; %bb.0:
; GFX1250-SDAG-NEXT:    global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-SDAG-NEXT:    v_nop
; GFX1250-SDAG-NEXT:    s_clause 0x1
; GFX1250-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10 nv
; GFX1250-SDAG-NEXT:    s_load_b32 s12, s[4:5], 0x20 nv
; GFX1250-SDAG-NEXT:    v_mov_b32_e32 v4, 0
; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT:    s_buffer_load_b128 s[8:11], s[0:3], s12 offset:0x0 nv
; GFX1250-SDAG-NEXT:    s_load_b64 s[6:7], s[4:5], 0x0 nv
; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
; GFX1250-SDAG-NEXT:    global_store_b128 v4, v[0:3], s[6:7]
; GFX1250-SDAG-NEXT:    s_endpgm
;
; GFX1250-GISEL-LABEL: ptr_s_buffer_load_v4i32:
; GFX1250-GISEL:       ; %bb.0:
; GFX1250-GISEL-NEXT:    global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-GISEL-NEXT:    v_nop
; GFX1250-GISEL-NEXT:    s_clause 0x2
; GFX1250-GISEL-NEXT:    s_load_b32 s8, s[4:5], 0x20 nv
; GFX1250-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10 nv
; GFX1250-GISEL-NEXT:    s_load_b64 s[6:7], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT:    v_mov_b32_e32 v4, 0
; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT:    s_buffer_load_b128 s[0:3], s[0:3], s8 offset:0x0 nv
; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT:    global_store_b128 v4, v[0:3], s[6:7]
; GFX1250-GISEL-NEXT:    s_endpgm
  %load = call <4 x i32> @llvm.amdgcn.ptr.s.buffer.load.v4i32(ptr addrspace(8) %rsrc, i32 %offset, i32 0), !invariant.load !0
  store <4 x i32> %load, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @ptr_s_buffer_load_i32_imm_offset(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc) {
;
; GFX6-LABEL: ptr_s_buffer_load_i32_imm_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    s_buffer_load_dword s4, s[0:3], 0x4
; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
; GFX6-NEXT:    s_mov_b32 s2, -1
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_mov_b32_e32 v0, s4
; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: ptr_s_buffer_load_i32_imm_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX7-NEXT:    s_add_i32 s12, s12, s17
; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    s_buffer_load_dword s0, s[0:3], 0x4
; GFX7-NEXT:    v_mov_b32_e32 v0, s4
; GFX7-NEXT:    v_mov_b32_e32 v1, s5
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    v_mov_b32_e32 v2, s0
; GFX7-NEXT:    flat_store_dword v[0:1], v2
; GFX7-NEXT:    s_endpgm
;
; GFX8-LABEL: ptr_s_buffer_load_i32_imm_offset:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX8-NEXT:    s_add_i32 s12, s12, s17
; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    s_buffer_load_dword s0, s[0:3], 0x10
; GFX8-NEXT:    v_mov_b32_e32 v0, s4
; GFX8-NEXT:    v_mov_b32_e32 v1, s5
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v2, s0
; GFX8-NEXT:    flat_store_dword v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: ptr_s_buffer_load_i32_imm_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX9-NEXT:    v_mov_b32_e32 v0, 0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_buffer_load_dword s6, s[0:3], 0x10
; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_mov_b32_e32 v1, s6
; GFX9-NEXT:    global_store_dword v0, v1, s[4:5]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: ptr_s_buffer_load_i32_imm_offset:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX10-NEXT:    v_mov_b32_e32 v0, 0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_buffer_load_dword s6, s[0:3], 0x10
; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v1, s6
; GFX10-NEXT:    global_store_dword v0, v1, s[4:5]
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: ptr_s_buffer_load_i32_imm_offset:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT:    v_mov_b32_e32 v0, 0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_buffer_load_b32 s2, s[0:3], 0x10
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_mov_b32_e32 v1, s2
; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT:    s_endpgm
;
; GFX1200-SDAG-LABEL: ptr_s_buffer_load_i32_imm_offset:
; GFX1200-SDAG:       ; %bb.0:
; GFX1200-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX1200-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT:    s_buffer_load_b32 s2, s[0:3], 0x10
; GFX1200-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
; GFX1200-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1200-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX1200-SDAG-NEXT:    s_endpgm
;
; GFX1200-GISEL-LABEL: ptr_s_buffer_load_i32_imm_offset:
; GFX1200-GISEL:       ; %bb.0:
; GFX1200-GISEL-NEXT:    s_clause 0x1
; GFX1200-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX1200-GISEL-NEXT:    s_load_b64 s[4:5], s[4:5], 0x0
; GFX1200-GISEL-NEXT:    v_mov_b32_e32 v1, 0
; GFX1200-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1200-GISEL-NEXT:    s_buffer_load_b32 s0, s[0:3], 0x10
; GFX1200-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1200-GISEL-NEXT:    v_mov_b32_e32 v0, s0
; GFX1200-GISEL-NEXT:    global_store_b32 v1, v0, s[4:5]
; GFX1200-GISEL-NEXT:    s_endpgm
;
; GFX1250-SDAG-LABEL: ptr_s_buffer_load_i32_imm_offset:
; GFX1250-SDAG:       ; %bb.0:
; GFX1250-SDAG-NEXT:    global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-SDAG-NEXT:    v_nop
; GFX1250-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10 nv
; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT:    s_buffer_load_b32 s8, s[0:3], 0x10 nv
; GFX1250-SDAG-NEXT:    s_load_b64 s[6:7], s[4:5], 0x0 nv
; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s8
; GFX1250-SDAG-NEXT:    global_store_b32 v0, v1, s[6:7]
; GFX1250-SDAG-NEXT:    s_endpgm
;
; GFX1250-GISEL-LABEL: ptr_s_buffer_load_i32_imm_offset:
; GFX1250-GISEL:       ; %bb.0:
; GFX1250-GISEL-NEXT:    global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-GISEL-NEXT:    v_nop
; GFX1250-GISEL-NEXT:    s_clause 0x1
; GFX1250-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10 nv
; GFX1250-GISEL-NEXT:    s_load_b64 s[6:7], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT:    v_mov_b32_e32 v1, 0
; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT:    s_buffer_load_b32 s0, s[0:3], 0x10 nv
; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT:    v_mov_b32_e32 v0, s0
; GFX1250-GISEL-NEXT:    global_store_b32 v1, v0, s[6:7]
; GFX1250-GISEL-NEXT:    s_endpgm
  %load = call i32 @llvm.amdgcn.ptr.s.buffer.load.i32(ptr addrspace(8) %rsrc, i32 16, i32 0), !invariant.load !0
  store i32 %load, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @ptr_s_buffer_load_v8i32_divergent_offset_glc(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc) {
;
; GFX6-LABEL: ptr_s_buffer_load_v8i32_divergent_offset_glc:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_load_dwordx4 v[1:4], v0, s[0:3], 0 offen offset:16 glc
; GFX6-NEXT:    buffer_load_dwordx4 v[5:8], v0, s[0:3], 0 offen glc
; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
; GFX6-NEXT:    s_mov_b32 s2, -1
; GFX6-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(0)
; GFX6-NEXT:    buffer_store_dwordx4 v[1:4], off, s[0:3], 0 offset:16
; GFX6-NEXT:    s_waitcnt vmcnt(1)
; GFX6-NEXT:    buffer_store_dwordx4 v[5:8], off, s[0:3], 0
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: ptr_s_buffer_load_v8i32_divergent_offset_glc:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
; GFX7-NEXT:    s_add_i32 s12, s12, s17
; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_load_dwordx4 v[1:4], v0, s[0:3], 0 offen offset:16 glc
; GFX7-NEXT:    buffer_load_dwordx4 v[5:8], v0, s[0:3], 0 offen glc
; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    s_add_u32 s2, s0, 16
; GFX7-NEXT:    s_addc_u32 s3, s1, 0
; GFX7-NEXT:    v_mov_b32_e32 v12, s3
; GFX7-NEXT:    v_mov_b32_e32 v11, s2
; GFX7-NEXT:    v_mov_b32_e32 v10, s1
; GFX7-NEXT:    v_mov_b32_e32 v9, s0
; GFX7-NEXT:    s_waitcnt vmcnt(1)
; GFX7-NEXT:    flat_store_dwordx4 v[11:12], v[1:4]
; GFX7-NEXT:    s_waitcnt vmcnt(1)
; GFX7-NEXT:    flat_store_dwordx4 v[9:10], v[5:8]
; GFX7-NEXT:    s_endpgm
;
; GFX8-LABEL: ptr_s_buffer_load_v8i32_divergent_offset_glc:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX8-NEXT:    s_add_i32 s12, s12, s17
; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    buffer_load_dwordx4 v[1:4], v0, s[0:3], 0 offen offset:16 glc
; GFX8-NEXT:    buffer_load_dwordx4 v[5:8], v0, s[0:3], 0 offen glc
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    s_add_u32 s2, s0, 16
; GFX8-NEXT:    s_addc_u32 s3, s1, 0
; GFX8-NEXT:    v_mov_b32_e32 v12, s3
; GFX8-NEXT:    v_mov_b32_e32 v11, s2
; GFX8-NEXT:    v_mov_b32_e32 v10, s1
; GFX8-NEXT:    v_mov_b32_e32 v9, s0
; GFX8-NEXT:    s_waitcnt vmcnt(1)
; GFX8-NEXT:    flat_store_dwordx4 v[11:12], v[1:4]
; GFX8-NEXT:    s_waitcnt vmcnt(1)
; GFX8-NEXT:    flat_store_dwordx4 v[9:10], v[5:8]
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: ptr_s_buffer_load_v8i32_divergent_offset_glc:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_load_dwordx4 v[1:4], v0, s[0:3], 0 offen offset:16 glc
; GFX9-NEXT:    buffer_load_dwordx4 v[5:8], v0, s[0:3], 0 offen glc
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-NEXT:    v_mov_b32_e32 v0, 0
; GFX9-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(0)
; GFX9-NEXT:    global_store_dwordx4 v0, v[1:4], s[0:1] offset:16
; GFX9-NEXT:    s_waitcnt vmcnt(1)
; GFX9-NEXT:    global_store_dwordx4 v0, v[5:8], s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: ptr_s_buffer_load_v8i32_divergent_offset_glc:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_clause 0x1
; GFX10-NEXT:    buffer_load_dwordx4 v[1:4], v0, s[0:3], 0 offen offset:16 glc
; GFX10-NEXT:    buffer_load_dwordx4 v[5:8], v0, s[0:3], 0 offen glc
; GFX10-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX10-NEXT:    v_mov_b32_e32 v0, 0
; GFX10-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(0)
; GFX10-NEXT:    global_store_dwordx4 v0, v[1:4], s[0:1] offset:16
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    global_store_dwordx4 v0, v[5:8], s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: ptr_s_buffer_load_v8i32_divergent_offset_glc:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
; GFX11-NEXT:    v_mov_b32_e32 v8, 0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_clause 0x1
; GFX11-NEXT:    buffer_load_b128 v[0:3], v4, s[0:3], 0 offen offset:16 glc
; GFX11-NEXT:    buffer_load_b128 v[4:7], v4, s[0:3], 0 offen glc
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(0)
; GFX11-NEXT:    global_store_b128 v8, v[0:3], s[0:1] offset:16
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    global_store_b128 v8, v[4:7], s[0:1]
; GFX11-NEXT:    s_endpgm
;
; GFX1200-SDAG-LABEL: ptr_s_buffer_load_v8i32_divergent_offset_glc:
; GFX1200-SDAG:       ; %bb.0:
; GFX1200-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX1200-SDAG-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
; GFX1200-SDAG-NEXT:    v_mov_b32_e32 v8, 0
; GFX1200-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT:    s_clause 0x1
; GFX1200-SDAG-NEXT:    buffer_load_b128 v[0:3], v4, s[0:3], null offen offset:16 th:TH_LOAD_NT
; GFX1200-SDAG-NEXT:    buffer_load_b128 v[4:7], v4, s[0:3], null offen th:TH_LOAD_NT
; GFX1200-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
; GFX1200-SDAG-NEXT:    s_wait_loadcnt 0x1
; GFX1200-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT:    global_store_b128 v8, v[0:3], s[0:1] offset:16
; GFX1200-SDAG-NEXT:    s_wait_loadcnt 0x0
; GFX1200-SDAG-NEXT:    global_store_b128 v8, v[4:7], s[0:1]
; GFX1200-SDAG-NEXT:    s_endpgm
;
; GFX1200-GISEL-LABEL: ptr_s_buffer_load_v8i32_divergent_offset_glc:
; GFX1200-GISEL:       ; %bb.0:
; GFX1200-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX1200-GISEL-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
; GFX1200-GISEL-NEXT:    s_load_b64 s[4:5], s[4:5], 0x0
; GFX1200-GISEL-NEXT:    v_mov_b32_e32 v8, 0
; GFX1200-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1200-GISEL-NEXT:    s_clause 0x1
; GFX1200-GISEL-NEXT:    buffer_load_b128 v[0:3], v4, s[0:3], null offen th:TH_LOAD_NT
; GFX1200-GISEL-NEXT:    buffer_load_b128 v[4:7], v4, s[0:3], null offen offset:16 th:TH_LOAD_NT
; GFX1200-GISEL-NEXT:    s_wait_loadcnt 0x1
; GFX1200-GISEL-NEXT:    global_store_b128 v8, v[0:3], s[4:5]
; GFX1200-GISEL-NEXT:    s_wait_loadcnt 0x0
; GFX1200-GISEL-NEXT:    global_store_b128 v8, v[4:7], s[4:5] offset:16
; GFX1200-GISEL-NEXT:    s_endpgm
;
; GFX1250-SDAG-LABEL: ptr_s_buffer_load_v8i32_divergent_offset_glc:
; GFX1250-SDAG:       ; %bb.0:
; GFX1250-SDAG-NEXT:    global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-SDAG-NEXT:    v_nop
; GFX1250-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10 nv
; GFX1250-SDAG-NEXT:    v_and_b32_e32 v8, 0x3ff, v0
; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT:    s_clause 0x1
; GFX1250-SDAG-NEXT:    buffer_load_b128 v[0:3], v8, s[0:3], null offen offset:16 th:TH_LOAD_NT nv
; GFX1250-SDAG-NEXT:    buffer_load_b128 v[4:7], v8, s[0:3], null offen th:TH_LOAD_NT nv
; GFX1250-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-SDAG-NEXT:    v_mov_b32_e32 v8, 0
; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x1
; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT:    global_store_b128 v8, v[0:3], s[0:1] offset:16
; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT:    global_store_b128 v8, v[4:7], s[0:1]
; GFX1250-SDAG-NEXT:    s_endpgm
;
; GFX1250-GISEL-LABEL: ptr_s_buffer_load_v8i32_divergent_offset_glc:
; GFX1250-GISEL:       ; %bb.0:
; GFX1250-GISEL-NEXT:    global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-GISEL-NEXT:    v_nop
; GFX1250-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10 nv
; GFX1250-GISEL-NEXT:    v_and_b32_e32 v8, 0x3ff, v0
; GFX1250-GISEL-NEXT:    s_load_b64 s[4:5], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT:    s_clause 0x1
; GFX1250-GISEL-NEXT:    buffer_load_b128 v[0:3], v8, s[0:3], null offen th:TH_LOAD_NT nv
; GFX1250-GISEL-NEXT:    buffer_load_b128 v[4:7], v8, s[0:3], null offen offset:16 th:TH_LOAD_NT nv
; GFX1250-GISEL-NEXT:    v_mov_b32_e32 v8, 0
; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x1
; GFX1250-GISEL-NEXT:    global_store_b128 v8, v[0:3], s[4:5]
; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT:    global_store_b128 v8, v[4:7], s[4:5] offset:16
; GFX1250-GISEL-NEXT:    s_endpgm
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  %load = call <8 x i32> @llvm.amdgcn.ptr.s.buffer.load.v8i32(ptr addrspace(8) %rsrc, i32 %tid, i32 1), !invariant.load !0
  store <8 x i32> %load, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @ptr_s_buffer_load_v16i32_divergent_offset(ptr addrspace(1) %out, ptr addrspace(8) inreg %rsrc) {
;
; GFX6-LABEL: ptr_s_buffer_load_v16i32_divergent_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_load_dwordx4 v[1:4], v0, s[0:3], 0 offen offset:48
; GFX6-NEXT:    buffer_load_dwordx4 v[5:8], v0, s[0:3], 0 offen offset:32
; GFX6-NEXT:    buffer_load_dwordx4 v[9:12], v0, s[0:3], 0 offen offset:16
; GFX6-NEXT:    buffer_load_dwordx4 v[13:16], v0, s[0:3], 0 offen
; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
; GFX6-NEXT:    s_mov_b32 s2, -1
; GFX6-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(0)
; GFX6-NEXT:    buffer_store_dwordx4 v[1:4], off, s[0:3], 0 offset:48
; GFX6-NEXT:    s_waitcnt vmcnt(3)
; GFX6-NEXT:    buffer_store_dwordx4 v[5:8], off, s[0:3], 0 offset:32
; GFX6-NEXT:    s_waitcnt vmcnt(3)
; GFX6-NEXT:    buffer_store_dwordx4 v[9:12], off, s[0:3], 0 offset:16
; GFX6-NEXT:    s_waitcnt vmcnt(3)
; GFX6-NEXT:    buffer_store_dwordx4 v[13:16], off, s[0:3], 0
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: ptr_s_buffer_load_v16i32_divergent_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x4
; GFX7-NEXT:    s_add_i32 s12, s12, s17
; GFX7-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GFX7-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_load_dwordx4 v[1:4], v0, s[0:3], 0 offen offset:48
; GFX7-NEXT:    buffer_load_dwordx4 v[5:8], v0, s[0:3], 0 offen offset:32
; GFX7-NEXT:    buffer_load_dwordx4 v[9:12], v0, s[0:3], 0 offen offset:16
; GFX7-NEXT:    buffer_load_dwordx4 v[13:16], v0, s[0:3], 0 offen
; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    s_add_u32 s2, s0, 48
; GFX7-NEXT:    s_addc_u32 s3, s1, 0
; GFX7-NEXT:    v_mov_b32_e32 v20, s3
; GFX7-NEXT:    v_mov_b32_e32 v19, s2
; GFX7-NEXT:    s_add_u32 s2, s0, 32
; GFX7-NEXT:    s_addc_u32 s3, s1, 0
; GFX7-NEXT:    v_mov_b32_e32 v18, s1
; GFX7-NEXT:    v_mov_b32_e32 v17, s0
; GFX7-NEXT:    s_add_u32 s0, s0, 16
; GFX7-NEXT:    v_mov_b32_e32 v0, s2
; GFX7-NEXT:    s_addc_u32 s1, s1, 0
; GFX7-NEXT:    s_waitcnt vmcnt(3)
; GFX7-NEXT:    flat_store_dwordx4 v[19:20], v[1:4]
; GFX7-NEXT:    s_nop 0
; GFX7-NEXT:    v_mov_b32_e32 v1, s3
; GFX7-NEXT:    v_mov_b32_e32 v3, s1
; GFX7-NEXT:    v_mov_b32_e32 v2, s0
; GFX7-NEXT:    s_waitcnt vmcnt(3)
; GFX7-NEXT:    flat_store_dwordx4 v[0:1], v[5:8]
; GFX7-NEXT:    s_waitcnt vmcnt(3)
; GFX7-NEXT:    flat_store_dwordx4 v[2:3], v[9:12]
; GFX7-NEXT:    s_waitcnt vmcnt(3)
; GFX7-NEXT:    flat_store_dwordx4 v[17:18], v[13:16]
; GFX7-NEXT:    s_endpgm
;
; GFX8-LABEL: ptr_s_buffer_load_v16i32_divergent_offset:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX8-NEXT:    s_add_i32 s12, s12, s17
; GFX8-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
; GFX8-NEXT:    s_mov_b32 flat_scratch_lo, s13
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    buffer_load_dwordx4 v[1:4], v0, s[0:3], 0 offen offset:48
; GFX8-NEXT:    buffer_load_dwordx4 v[5:8], v0, s[0:3], 0 offen offset:32
; GFX8-NEXT:    buffer_load_dwordx4 v[9:12], v0, s[0:3], 0 offen offset:16
; GFX8-NEXT:    buffer_load_dwordx4 v[13:16], v0, s[0:3], 0 offen
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    s_add_u32 s2, s0, 48
; GFX8-NEXT:    s_addc_u32 s3, s1, 0
; GFX8-NEXT:    v_mov_b32_e32 v20, s3
; GFX8-NEXT:    v_mov_b32_e32 v19, s2
; GFX8-NEXT:    s_add_u32 s2, s0, 32
; GFX8-NEXT:    s_addc_u32 s3, s1, 0
; GFX8-NEXT:    v_mov_b32_e32 v18, s1
; GFX8-NEXT:    v_mov_b32_e32 v17, s0
; GFX8-NEXT:    s_add_u32 s0, s0, 16
; GFX8-NEXT:    v_mov_b32_e32 v0, s2
; GFX8-NEXT:    s_addc_u32 s1, s1, 0
; GFX8-NEXT:    s_waitcnt vmcnt(3)
; GFX8-NEXT:    flat_store_dwordx4 v[19:20], v[1:4]
; GFX8-NEXT:    s_nop 0
; GFX8-NEXT:    v_mov_b32_e32 v1, s3
; GFX8-NEXT:    v_mov_b32_e32 v3, s1
; GFX8-NEXT:    v_mov_b32_e32 v2, s0
; GFX8-NEXT:    s_waitcnt vmcnt(3)
; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[5:8]
; GFX8-NEXT:    s_waitcnt vmcnt(3)
; GFX8-NEXT:    flat_store_dwordx4 v[2:3], v[9:12]
; GFX8-NEXT:    s_waitcnt vmcnt(3)
; GFX8-NEXT:    flat_store_dwordx4 v[17:18], v[13:16]
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: ptr_s_buffer_load_v16i32_divergent_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_load_dwordx4 v[1:4], v0, s[0:3], 0 offen offset:48
; GFX9-NEXT:    buffer_load_dwordx4 v[5:8], v0, s[0:3], 0 offen offset:32
; GFX9-NEXT:    buffer_load_dwordx4 v[9:12], v0, s[0:3], 0 offen offset:16
; GFX9-NEXT:    buffer_load_dwordx4 v[13:16], v0, s[0:3], 0 offen
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-NEXT:    v_mov_b32_e32 v0, 0
; GFX9-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(0)
; GFX9-NEXT:    global_store_dwordx4 v0, v[1:4], s[0:1] offset:48
; GFX9-NEXT:    s_waitcnt vmcnt(3)
; GFX9-NEXT:    global_store_dwordx4 v0, v[5:8], s[0:1] offset:32
; GFX9-NEXT:    s_waitcnt vmcnt(3)
; GFX9-NEXT:    global_store_dwordx4 v0, v[9:12], s[0:1] offset:16
; GFX9-NEXT:    s_waitcnt vmcnt(3)
; GFX9-NEXT:    global_store_dwordx4 v0, v[13:16], s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: ptr_s_buffer_load_v16i32_divergent_offset:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_clause 0x3
; GFX10-NEXT:    buffer_load_dwordx4 v[1:4], v0, s[0:3], 0 offen offset:48
; GFX10-NEXT:    buffer_load_dwordx4 v[5:8], v0, s[0:3], 0 offen offset:32
; GFX10-NEXT:    buffer_load_dwordx4 v[9:12], v0, s[0:3], 0 offen offset:16
; GFX10-NEXT:    buffer_load_dwordx4 v[13:16], v0, s[0:3], 0 offen
; GFX10-NEXT:    s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX10-NEXT:    v_mov_b32_e32 v0, 0
; GFX10-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(0)
; GFX10-NEXT:    global_store_dwordx4 v0, v[1:4], s[0:1] offset:48
; GFX10-NEXT:    s_waitcnt vmcnt(2)
; GFX10-NEXT:    global_store_dwordx4 v0, v[5:8], s[0:1] offset:32
; GFX10-NEXT:    s_waitcnt vmcnt(1)
; GFX10-NEXT:    global_store_dwordx4 v0, v[9:12], s[0:1] offset:16
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    global_store_dwordx4 v0, v[13:16], s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: ptr_s_buffer_load_v16i32_divergent_offset:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT:    v_and_b32_e32 v12, 0x3ff, v0
; GFX11-NEXT:    v_mov_b32_e32 v16, 0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_clause 0x3
; GFX11-NEXT:    buffer_load_b128 v[0:3], v12, s[0:3], 0 offen offset:48
; GFX11-NEXT:    buffer_load_b128 v[4:7], v12, s[0:3], 0 offen offset:32
; GFX11-NEXT:    buffer_load_b128 v[8:11], v12, s[0:3], 0 offen offset:16
; GFX11-NEXT:    buffer_load_b128 v[12:15], v12, s[0:3], 0 offen
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(0)
; GFX11-NEXT:    global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX11-NEXT:    s_waitcnt vmcnt(2)
; GFX11-NEXT:    global_store_b128 v16, v[4:7], s[0:1] offset:32
; GFX11-NEXT:    s_waitcnt vmcnt(1)
; GFX11-NEXT:    global_store_b128 v16, v[8:11], s[0:1] offset:16
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    global_store_b128 v16, v[12:15], s[0:1]
; GFX11-NEXT:    s_endpgm
;
; GFX1200-SDAG-LABEL: ptr_s_buffer_load_v16i32_divergent_offset:
; GFX1200-SDAG:       ; %bb.0:
; GFX1200-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX1200-SDAG-NEXT:    v_and_b32_e32 v12, 0x3ff, v0
; GFX1200-SDAG-NEXT:    v_mov_b32_e32 v16, 0
; GFX1200-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT:    s_clause 0x3
; GFX1200-SDAG-NEXT:    buffer_load_b128 v[0:3], v12, s[0:3], null offen offset:48
; GFX1200-SDAG-NEXT:    buffer_load_b128 v[4:7], v12, s[0:3], null offen offset:32
; GFX1200-SDAG-NEXT:    buffer_load_b128 v[8:11], v12, s[0:3], null offen offset:16
; GFX1200-SDAG-NEXT:    buffer_load_b128 v[12:15], v12, s[0:3], null offen
; GFX1200-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
; GFX1200-SDAG-NEXT:    s_wait_loadcnt 0x3
; GFX1200-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT:    global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX1200-SDAG-NEXT:    s_wait_loadcnt 0x2
; GFX1200-SDAG-NEXT:    global_store_b128 v16, v[4:7], s[0:1] offset:32
; GFX1200-SDAG-NEXT:    s_wait_loadcnt 0x1
; GFX1200-SDAG-NEXT:    global_store_b128 v16, v[8:11], s[0:1] offset:16
; GFX1200-SDAG-NEXT:    s_wait_loadcnt 0x0
; GFX1200-SDAG-NEXT:    global_store_b128 v16, v[12:15], s[0:1]
; GFX1200-SDAG-NEXT:    s_endpgm
;
; GFX1200-GISEL-LABEL: ptr_s_buffer_load_v16i32_divergent_offset:
; GFX1200-GISEL:       ; %bb.0:
; GFX1200-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10
; GFX1200-GISEL-NEXT:    v_and_b32_e32 v12, 0x3ff, v0
; GFX1200-GISEL-NEXT:    s_load_b64 s[4:5], s[4:5], 0x0
; GFX1200-GISEL-NEXT:    v_mov_b32_e32 v16, 0
; GFX1200-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1200-GISEL-NEXT:    s_clause 0x3
; GFX1200-GISEL-NEXT:    buffer_load_b128 v[0:3], v12, s[0:3], null offen
; GFX1200-GISEL-NEXT:    buffer_load_b128 v[4:7], v12, s[0:3], null offen offset:16
; GFX1200-GISEL-NEXT:    buffer_load_b128 v[8:11], v12, s[0:3], null offen offset:32
; GFX1200-GISEL-NEXT:    buffer_load_b128 v[12:15], v12, s[0:3], null offen offset:48
; GFX1200-GISEL-NEXT:    s_wait_loadcnt 0x3
; GFX1200-GISEL-NEXT:    global_store_b128 v16, v[0:3], s[4:5]
; GFX1200-GISEL-NEXT:    s_wait_loadcnt 0x2
; GFX1200-GISEL-NEXT:    global_store_b128 v16, v[4:7], s[4:5] offset:16
; GFX1200-GISEL-NEXT:    s_wait_loadcnt 0x1
; GFX1200-GISEL-NEXT:    global_store_b128 v16, v[8:11], s[4:5] offset:32
; GFX1200-GISEL-NEXT:    s_wait_loadcnt 0x0
; GFX1200-GISEL-NEXT:    global_store_b128 v16, v[12:15], s[4:5] offset:48
; GFX1200-GISEL-NEXT:    s_endpgm
;
; GFX1250-SDAG-LABEL: ptr_s_buffer_load_v16i32_divergent_offset:
; GFX1250-SDAG:       ; %bb.0:
; GFX1250-SDAG-NEXT:    global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-SDAG-NEXT:    v_nop
; GFX1250-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10 nv
; GFX1250-SDAG-NEXT:    v_and_b32_e32 v16, 0x3ff, v0
; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT:    s_clause 0x3
; GFX1250-SDAG-NEXT:    buffer_load_b128 v[0:3], v16, s[0:3], null offen offset:48 nv
; GFX1250-SDAG-NEXT:    buffer_load_b128 v[4:7], v16, s[0:3], null offen offset:32 nv
; GFX1250-SDAG-NEXT:    buffer_load_b128 v[8:11], v16, s[0:3], null offen offset:16 nv
; GFX1250-SDAG-NEXT:    buffer_load_b128 v[12:15], v16, s[0:3], null offen nv
; GFX1250-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-SDAG-NEXT:    v_mov_b32_e32 v16, 0
; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x3
; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT:    global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x2
; GFX1250-SDAG-NEXT:    global_store_b128 v16, v[4:7], s[0:1] offset:32
; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x1
; GFX1250-SDAG-NEXT:    global_store_b128 v16, v[8:11], s[0:1] offset:16
; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT:    global_store_b128 v16, v[12:15], s[0:1]
; GFX1250-SDAG-NEXT:    s_endpgm
;
; GFX1250-GISEL-LABEL: ptr_s_buffer_load_v16i32_divergent_offset:
; GFX1250-GISEL:       ; %bb.0:
; GFX1250-GISEL-NEXT:    global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-GISEL-NEXT:    v_nop
; GFX1250-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x10 nv
; GFX1250-GISEL-NEXT:    v_and_b32_e32 v16, 0x3ff, v0
; GFX1250-GISEL-NEXT:    s_load_b64 s[4:5], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT:    s_clause 0x3
; GFX1250-GISEL-NEXT:    buffer_load_b128 v[0:3], v16, s[0:3], null offen nv
; GFX1250-GISEL-NEXT:    buffer_load_b128 v[4:7], v16, s[0:3], null offen offset:16 nv
; GFX1250-GISEL-NEXT:    buffer_load_b128 v[8:11], v16, s[0:3], null offen offset:32 nv
; GFX1250-GISEL-NEXT:    buffer_load_b128 v[12:15], v16, s[0:3], null offen offset:48 nv
; GFX1250-GISEL-NEXT:    v_mov_b32_e32 v16, 0
; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x3
; GFX1250-GISEL-NEXT:    global_store_b128 v16, v[0:3], s[4:5]
; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x2
; GFX1250-GISEL-NEXT:    global_store_b128 v16, v[4:7], s[4:5] offset:16
; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x1
; GFX1250-GISEL-NEXT:    global_store_b128 v16, v[8:11], s[4:5] offset:32
; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT:    global_store_b128 v16, v[12:15], s[4:5] offset:48
; GFX1250-GISEL-NEXT:    s_endpgm
  %tid = call i32 @llvm.amdgcn.workitem.id.x()
  %load = call <16 x i32> @llvm.amdgcn.ptr.s.buffer.load.v16i32(ptr addrspace(8) %rsrc, i32 %tid, i32 0), !invariant.load !0
  store <16 x i32> %load, ptr addrspace(1) %out
  ret void
}

declare i32 @llvm.amdgcn.ptr.s.buffer.load.i32(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
declare <2 x i32> @llvm.amdgcn.ptr.s.buffer.load.v2i32(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
declare <4 x i32> @llvm.amdgcn.ptr.s.buffer.load.v4i32(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
declare <8 x i32> @llvm.amdgcn.ptr.s.buffer.load.v8i32(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
declare <16 x i32> @llvm.amdgcn.ptr.s.buffer.load.v16i32(ptr addrspace(8) nocapture readonly, i32, i32 immarg)
declare i32 @llvm.amdgcn.workitem.id.x()

!0 = !{}