; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgpu7.04 -enable-amdgpu-aa=0 -enable-misched -enable-aa-sched-mi < %s | FileCheck -enable-var-scope -check-prefix=CI %s
; RUN: llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgpu9.00 -enable-amdgpu-aa=0 -enable-misched -enable-aa-sched-mi < %s | FileCheck -enable-var-scope -check-prefix=GFX9 %s

%struct.lds = type { [64 x ptr], [16 x i8] }
@stored_lds_struct = addrspace(3) global %struct.lds poison, align 16
@stored_lds_ptr = addrspace(3) global ptr addrspace(3) poison, align 4
@stored_constant_ptr = addrspace(3) global ptr addrspace(4) poison, align 8
@stored_global_ptr = addrspace(3) global ptr addrspace(1) poison, align 8

define amdgpu_kernel void @no_reorder_flat_load_local_store_local_load(ptr addrspace(3) %out, ptr %fptr) #0 {
; CI-LABEL: no_reorder_flat_load_local_store_local_load:
; CI:       ; %bb.0:
; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb
; CI-NEXT:    v_mov_b32_e32 v4, 0
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    v_mov_b32_e32 v0, s0
; CI-NEXT:    v_mov_b32_e32 v1, s1
; CI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
; CI-NEXT:    s_load_dword s0, s[4:5], 0x9
; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT:    ds_write_b128 v4, v[0:3] offset:512
; CI-NEXT:    ds_read2_b32 v[0:1], v4 offset0:129 offset1:130
; CI-NEXT:    v_mov_b32_e32 v2, s0
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    ds_write2_b32 v2, v0, v1 offset1:1
; CI-NEXT:    s_endpgm
;
; GFX9-LABEL: no_reorder_flat_load_local_store_local_load:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c
; GFX9-NEXT:    v_mov_b32_e32 v4, 0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_mov_b32_e32 v0, s0
; GFX9-NEXT:    v_mov_b32_e32 v1, s1
; GFX9-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x24
; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_write_b128 v4, v[0:3] offset:512
; GFX9-NEXT:    ds_read2_b32 v[0:1], v4 offset0:129 offset1:130
; GFX9-NEXT:    v_mov_b32_e32 v2, s0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    ds_write2_b32 v2, v0, v1 offset1:1
; GFX9-NEXT:    s_endpgm
  %ptr1 = getelementptr %struct.lds, ptr addrspace(3) @stored_lds_struct, i32 0, i32 1
  %ptr2 = getelementptr %struct.lds, ptr addrspace(3) @stored_lds_struct, i32 0, i32 1, i32 4
  call void @llvm.memcpy.p3.p0(ptr addrspace(3) align 16 %ptr1, ptr align 8 %fptr, i64 16, i1 false)
  %vector_load = load <2 x i32>, ptr addrspace(3) %ptr2, align 4
  store <2 x i32> %vector_load, ptr addrspace(3) %out, align 4
  ret void
}

define amdgpu_kernel void @reorder_local_load_global_store_local_load(ptr addrspace(1) %out, ptr addrspace(1) %gptr) #0 {
; CI-LABEL: reorder_local_load_global_store_local_load:
; CI:       ; %bb.0:
; CI-NEXT:    v_mov_b32_e32 v0, 0
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_b32 v0, v0
; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT:    s_mov_b32 s7, 0xf000
; CI-NEXT:    s_mov_b32 s6, -1
; CI-NEXT:    v_mov_b32_e32 v2, 0x63
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    ds_read2_b32 v[0:1], v0 offset0:1 offset1:3
; CI-NEXT:    s_mov_b32 s4, s0
; CI-NEXT:    s_mov_b32 s5, s1
; CI-NEXT:    s_mov_b32 s0, s2
; CI-NEXT:    s_mov_b32 s1, s3
; CI-NEXT:    s_mov_b32 s2, s6
; CI-NEXT:    s_mov_b32 s3, s7
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
; CI-NEXT:    buffer_store_dword v2, off, s[0:3], 0
; CI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT:    s_endpgm
;
; GFX9-LABEL: reorder_local_load_global_store_local_load:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    v_mov_b32_e32 v2, 0
; GFX9-NEXT:    ds_read_b32 v0, v2
; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT:    v_mov_b32_e32 v3, 0x63
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    ds_read2_b32 v[0:1], v0 offset0:1 offset1:3
; GFX9-NEXT:    global_store_dword v2, v3, s[2:3]
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]
; GFX9-NEXT:    s_endpgm
  %ptr0 = load ptr addrspace(3), ptr addrspace(3) @stored_lds_ptr, align 4

  %ptr1 = getelementptr inbounds i32, ptr addrspace(3) %ptr0, i32 1
  %ptr2 = getelementptr inbounds i32, ptr addrspace(3) %ptr0, i32 3

  %tmp1 = load i32, ptr addrspace(3) %ptr1, align 4
  store i32 99, ptr addrspace(1) %gptr, align 4
  %tmp2 = load i32, ptr addrspace(3) %ptr2, align 4

  %add = add nsw i32 %tmp1, %tmp2

  store i32 %add, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_kernel void @no_reorder_local_load_volatile_global_store_local_load(ptr addrspace(1) %out, ptr addrspace(1) %gptr) #0 {
; CI-LABEL: no_reorder_local_load_volatile_global_store_local_load:
; CI:       ; %bb.0:
; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT:    v_mov_b32_e32 v0, 0
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_b32 v0, v0
; CI-NEXT:    s_mov_b32 s7, 0xf000
; CI-NEXT:    s_mov_b32 s6, -1
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_mov_b32 s8, s2
; CI-NEXT:    s_mov_b32 s9, s3
; CI-NEXT:    s_mov_b32 s10, s6
; CI-NEXT:    s_mov_b32 s11, s7
; CI-NEXT:    v_mov_b32_e32 v2, 0x63
; CI-NEXT:    ds_read_b32 v1, v0 offset:4
; CI-NEXT:    buffer_store_dword v2, off, s[8:11], 0
; CI-NEXT:    s_waitcnt vmcnt(0)
; CI-NEXT:    ds_read_b32 v0, v0 offset:12
; CI-NEXT:    s_mov_b32 s4, s0
; CI-NEXT:    s_mov_b32 s5, s1
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
; CI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT:    s_endpgm
;
; GFX9-LABEL: no_reorder_local_load_volatile_global_store_local_load:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    v_mov_b32_e32 v0, 0
; GFX9-NEXT:    ds_read_b32 v1, v0
; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT:    v_mov_b32_e32 v2, 0x63
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    ds_read_b32 v3, v1 offset:4
; GFX9-NEXT:    global_store_dword v0, v2, s[2:3]
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    ds_read_b32 v1, v1 offset:12
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_u32_e32 v1, v3, v1
; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
  %ptr0 = load ptr addrspace(3), ptr addrspace(3) @stored_lds_ptr, align 4

  %ptr1 = getelementptr inbounds i32, ptr addrspace(3) %ptr0, i32 1
  %ptr2 = getelementptr inbounds i32, ptr addrspace(3) %ptr0, i32 3

  %tmp1 = load i32, ptr addrspace(3) %ptr1, align 4
  store volatile i32 99, ptr addrspace(1) %gptr, align 4
  %tmp2 = load i32, ptr addrspace(3) %ptr2, align 4

  %add = add nsw i32 %tmp1, %tmp2

  store i32 %add, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_kernel void @no_reorder_barrier_local_load_global_store_local_load(ptr addrspace(1) %out, ptr addrspace(1) %gptr) #0 {
; CI-LABEL: no_reorder_barrier_local_load_global_store_local_load:
; CI:       ; %bb.0:
; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT:    v_mov_b32_e32 v0, 0
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_b32 v0, v0
; CI-NEXT:    s_mov_b32 s7, 0xf000
; CI-NEXT:    s_mov_b32 s6, -1
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_mov_b32 s8, s2
; CI-NEXT:    s_mov_b32 s9, s3
; CI-NEXT:    s_mov_b32 s10, s6
; CI-NEXT:    s_mov_b32 s11, s7
; CI-NEXT:    v_mov_b32_e32 v2, 0x63
; CI-NEXT:    ds_read_b32 v1, v0 offset:4
; CI-NEXT:    buffer_store_dword v2, off, s[8:11], 0
; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; CI-NEXT:    s_barrier
; CI-NEXT:    ds_read_b32 v0, v0 offset:12
; CI-NEXT:    s_mov_b32 s4, s0
; CI-NEXT:    s_mov_b32 s5, s1
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
; CI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT:    s_endpgm
;
; GFX9-LABEL: no_reorder_barrier_local_load_global_store_local_load:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    v_mov_b32_e32 v0, 0
; GFX9-NEXT:    ds_read_b32 v1, v0
; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT:    v_mov_b32_e32 v2, 0x63
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    ds_read_b32 v3, v1 offset:4
; GFX9-NEXT:    global_store_dword v0, v2, s[2:3]
; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_barrier
; GFX9-NEXT:    ds_read_b32 v1, v1 offset:12
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_add_u32_e32 v1, v3, v1
; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
  %ptr0 = load ptr addrspace(3), ptr addrspace(3) @stored_lds_ptr, align 4

  %ptr1 = getelementptr inbounds i32, ptr addrspace(3) %ptr0, i32 1
  %ptr2 = getelementptr inbounds i32, ptr addrspace(3) %ptr0, i32 3

  %tmp1 = load i32, ptr addrspace(3) %ptr1, align 4
  store i32 99, ptr addrspace(1) %gptr, align 4
  call void @llvm.amdgcn.s.barrier() #1
  %tmp2 = load i32, ptr addrspace(3) %ptr2, align 4

  %add = add nsw i32 %tmp1, %tmp2

  store i32 %add, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_kernel void @reorder_constant_load_global_store_constant_load(ptr addrspace(1) %out, ptr addrspace(1) %gptr) #0 {
; CI-LABEL: reorder_constant_load_global_store_constant_load:
; CI:       ; %bb.0:
; CI-NEXT:    v_mov_b32_e32 v0, 0
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT:    ds_read_b64 v[0:1], v0
; CI-NEXT:    s_mov_b32 s7, 0xf000
; CI-NEXT:    s_mov_b32 s6, -1
; CI-NEXT:    s_mov_b32 s10, s6
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_mov_b32 s8, s2
; CI-NEXT:    s_mov_b32 s9, s3
; CI-NEXT:    s_mov_b32 s11, s7
; CI-NEXT:    v_readfirstlane_b32 s2, v0
; CI-NEXT:    v_readfirstlane_b32 s3, v1
; CI-NEXT:    v_mov_b32_e32 v0, 0x63
; CI-NEXT:    s_load_dword s12, s[2:3], 0x1
; CI-NEXT:    buffer_store_dword v0, off, s[8:11], 0
; CI-NEXT:    s_load_dword s2, s[2:3], 0x3
; CI-NEXT:    s_mov_b32 s4, s0
; CI-NEXT:    s_mov_b32 s5, s1
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_add_i32 s0, s12, s2
; CI-NEXT:    v_mov_b32_e32 v0, s0
; CI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT:    s_endpgm
;
; GFX9-LABEL: reorder_constant_load_global_store_constant_load:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    v_mov_b32_e32 v2, 0
; GFX9-NEXT:    ds_read_b64 v[0:1], v2
; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT:    v_mov_b32_e32 v3, 0x63
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
; GFX9-NEXT:    v_readfirstlane_b32 s5, v1
; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x4
; GFX9-NEXT:    global_store_dword v2, v3, s[2:3]
; GFX9-NEXT:    s_load_dword s2, s[4:5], 0xc
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_add_i32 s2, s6, s2
; GFX9-NEXT:    v_mov_b32_e32 v0, s2
; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]
; GFX9-NEXT:    s_endpgm
  %ptr0 = load ptr addrspace(4), ptr addrspace(3) @stored_constant_ptr, align 8

  %ptr1 = getelementptr inbounds i32, ptr addrspace(4) %ptr0, i64 1
  %ptr2 = getelementptr inbounds i32, ptr addrspace(4) %ptr0, i64 3

  %tmp1 = load i32, ptr addrspace(4) %ptr1, align 4
  store i32 99, ptr addrspace(1) %gptr, align 4
  %tmp2 = load i32, ptr addrspace(4) %ptr2, align 4

  %add = add nsw i32 %tmp1, %tmp2

  store i32 %add, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_kernel void @reorder_constant_load_local_store_constant_load(ptr addrspace(1) %out, ptr addrspace(3) %lptr) #0 {
; CI-LABEL: reorder_constant_load_local_store_constant_load:
; CI:       ; %bb.0:
; CI-NEXT:    v_mov_b32_e32 v0, 0
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_b64 v[0:1], v0
; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
; CI-NEXT:    s_load_dword s6, s[4:5], 0xb
; CI-NEXT:    s_mov_b32 s3, 0xf000
; CI-NEXT:    s_mov_b32 s2, -1
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    v_readfirstlane_b32 s4, v0
; CI-NEXT:    v_readfirstlane_b32 s5, v1
; CI-NEXT:    s_load_dword s7, s[4:5], 0x1
; CI-NEXT:    s_load_dword s4, s[4:5], 0x3
; CI-NEXT:    v_mov_b32_e32 v0, 0x63
; CI-NEXT:    v_mov_b32_e32 v1, s6
; CI-NEXT:    ds_write_b32 v1, v0
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_add_i32 s4, s7, s4
; CI-NEXT:    v_mov_b32_e32 v0, s4
; CI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
; CI-NEXT:    s_endpgm
;
; GFX9-LABEL: reorder_constant_load_local_store_constant_load:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    v_mov_b32_e32 v2, 0
; GFX9-NEXT:    ds_read_b64 v[0:1], v2
; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
; GFX9-NEXT:    s_load_dword s7, s[0:1], 0x4
; GFX9-NEXT:    s_load_dword s8, s[0:1], 0xc
; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
; GFX9-NEXT:    v_mov_b32_e32 v0, 0x63
; GFX9-NEXT:    v_mov_b32_e32 v1, s6
; GFX9-NEXT:    ds_write_b32 v1, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_add_i32 s0, s7, s8
; GFX9-NEXT:    v_mov_b32_e32 v0, s0
; GFX9-NEXT:    global_store_dword v2, v0, s[2:3]
; GFX9-NEXT:    s_endpgm
  %ptr0 = load ptr addrspace(4), ptr addrspace(3) @stored_constant_ptr, align 8

  %ptr1 = getelementptr inbounds i32, ptr addrspace(4) %ptr0, i64 1
  %ptr2 = getelementptr inbounds i32, ptr addrspace(4) %ptr0, i64 3

  %tmp1 = load i32, ptr addrspace(4) %ptr1, align 4
  store i32 99, ptr addrspace(3) %lptr, align 4
  %tmp2 = load i32, ptr addrspace(4) %ptr2, align 4

  %add = add nsw i32 %tmp1, %tmp2

  store i32 %add, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_kernel void @reorder_smrd_load_local_store_smrd_load(ptr addrspace(1) %out, ptr addrspace(3) noalias %lptr, ptr addrspace(4) %ptr0) #0 {
; CI-LABEL: reorder_smrd_load_local_store_smrd_load:
; CI:       ; %bb.0:
; CI-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
; CI-NEXT:    s_load_dword s8, s[4:5], 0xb
; CI-NEXT:    v_mov_b32_e32 v0, 0x63
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_load_dwordx2 s[4:5], s[6:7], 0x1
; CI-NEXT:    s_mov_b32 s3, 0xf000
; CI-NEXT:    v_mov_b32_e32 v1, s8
; CI-NEXT:    s_mov_b32 s2, -1
; CI-NEXT:    ds_write_b32 v1, v0
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_add_i32 s4, s4, s5
; CI-NEXT:    v_mov_b32_e32 v0, s4
; CI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
; CI-NEXT:    s_endpgm
;
; GFX9-LABEL: reorder_smrd_load_local_store_smrd_load:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c
; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
; GFX9-NEXT:    v_mov_b32_e32 v1, 0x63
; GFX9-NEXT:    v_mov_b32_e32 v0, 0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x4
; GFX9-NEXT:    v_mov_b32_e32 v2, s6
; GFX9-NEXT:    ds_write_b32 v2, v1
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_add_i32 s0, s4, s5
; GFX9-NEXT:    v_mov_b32_e32 v1, s0
; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
; GFX9-NEXT:    s_endpgm
  %ptr1 = getelementptr inbounds i32, ptr addrspace(4) %ptr0, i64 1
  %ptr2 = getelementptr inbounds i32, ptr addrspace(4) %ptr0, i64 2

  %tmp1 = load i32, ptr addrspace(4) %ptr1, align 4
  store i32 99, ptr addrspace(3) %lptr, align 4
  %tmp2 = load i32, ptr addrspace(4) %ptr2, align 4

  %add = add nsw i32 %tmp1, %tmp2

  store i32 %add, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_kernel void @reorder_global_load_local_store_global_load(ptr addrspace(1) %out, ptr addrspace(3) %lptr, ptr addrspace(1) %ptr0) #0 {
; CI-LABEL: reorder_global_load_local_store_global_load:
; CI:       ; %bb.0:
; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd
; CI-NEXT:    s_mov_b32 s7, 0xf000
; CI-NEXT:    s_mov_b32 s6, -1
; CI-NEXT:    s_mov_b32 s2, s6
; CI-NEXT:    s_mov_b32 s3, s7
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    buffer_load_dword v0, off, s[0:3], 0 offset:4
; CI-NEXT:    buffer_load_dword v1, off, s[0:3], 0 offset:12
; CI-NEXT:    s_load_dword s0, s[4:5], 0xb
; CI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
; CI-NEXT:    v_mov_b32_e32 v2, 0x63
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    v_mov_b32_e32 v3, s0
; CI-NEXT:    ds_write_b32 v3, v2
; CI-NEXT:    s_waitcnt vmcnt(0)
; CI-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
; CI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT:    s_endpgm
;
; GFX9-LABEL: reorder_global_load_local_store_global_load:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
; GFX9-NEXT:    v_mov_b32_e32 v0, 0
; GFX9-NEXT:    v_mov_b32_e32 v3, 0x63
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_dword v1, v0, s[0:1] offset:4
; GFX9-NEXT:    global_load_dword v2, v0, s[0:1] offset:12
; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_mov_b32_e32 v4, s2
; GFX9-NEXT:    ds_write_b32 v4, v3
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_add_u32_e32 v1, v1, v2
; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
  %ptr1 = getelementptr inbounds i32, ptr addrspace(1) %ptr0, i64 1
  %ptr2 = getelementptr inbounds i32, ptr addrspace(1) %ptr0, i64 3

  %tmp1 = load i32, ptr addrspace(1) %ptr1, align 4
  store i32 99, ptr addrspace(3) %lptr, align 4
  %tmp2 = load i32, ptr addrspace(1) %ptr2, align 4

  %add = add nsw i32 %tmp1, %tmp2

  store i32 %add, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_kernel void @reorder_local_offsets(ptr addrspace(1) nocapture %out, ptr addrspace(1) noalias nocapture readnone %gptr, ptr addrspace(3) noalias nocapture %ptr0) #0 {
; CI-LABEL: reorder_local_offsets:
; CI:       ; %bb.0:
; CI-NEXT:    s_load_dword s6, s[4:5], 0xd
; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    v_mov_b32_e32 v2, 0x7b
; CI-NEXT:    s_mov_b32 s3, 0xf000
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    v_mov_b32_e32 v3, s6
; CI-NEXT:    ds_read2_b32 v[0:1], v3 offset0:100 offset1:102
; CI-NEXT:    s_mov_b32 s2, -1
; CI-NEXT:    ds_write2_b32 v3, v2, v2 offset0:3 offset1:100
; CI-NEXT:    v_mov_b32_e32 v2, 0x315
; CI-NEXT:    ds_write_b32 v3, v2 offset:408
; CI-NEXT:    s_waitcnt lgkmcnt(2)
; CI-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
; CI-NEXT:    v_add_i32_e32 v0, vcc, 0x7b, v0
; CI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
; CI-NEXT:    s_endpgm
;
; GFX9-LABEL: reorder_local_offsets:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x34
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    v_mov_b32_e32 v4, 0x7b
; GFX9-NEXT:    v_mov_b32_e32 v2, 0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    v_mov_b32_e32 v3, s2
; GFX9-NEXT:    ds_read2_b32 v[0:1], v3 offset0:100 offset1:102
; GFX9-NEXT:    ds_write2_b32 v3, v4, v4 offset0:3 offset1:100
; GFX9-NEXT:    v_mov_b32_e32 v4, 0x315
; GFX9-NEXT:    ds_write_b32 v3, v4 offset:408
; GFX9-NEXT:    s_waitcnt lgkmcnt(2)
; GFX9-NEXT:    v_add_u32_e32 v0, v1, v0
; GFX9-NEXT:    v_add_u32_e32 v0, 0x7b, v0
; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]
; GFX9-NEXT:    s_endpgm
  %ptr1 = getelementptr inbounds i32, ptr addrspace(3) %ptr0, i32 3
  %ptr2 = getelementptr inbounds i32, ptr addrspace(3) %ptr0, i32 100
  %ptr3 = getelementptr inbounds i32, ptr addrspace(3) %ptr0, i32 102

  store i32 123, ptr addrspace(3) %ptr1, align 4
  %tmp1 = load i32, ptr addrspace(3) %ptr2, align 4
  %tmp2 = load i32, ptr addrspace(3) %ptr3, align 4
  store i32 123, ptr addrspace(3) %ptr2, align 4
  %tmp3 = load i32, ptr addrspace(3) %ptr1, align 4
  store i32 789, ptr addrspace(3) %ptr3, align 4

  %add.0 = add nsw i32 %tmp2, %tmp1
  %add.1 = add nsw i32 %add.0, %tmp3
  store i32 %add.1, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_kernel void @reorder_global_offsets(ptr addrspace(1) nocapture %out, ptr addrspace(1) noalias nocapture readnone %gptr, ptr addrspace(1) noalias nocapture %ptr0) #0 {
; CI-LABEL: reorder_global_offsets:
; CI:       ; %bb.0:
; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd
; CI-NEXT:    s_mov_b32 s7, 0xf000
; CI-NEXT:    s_mov_b32 s6, -1
; CI-NEXT:    s_mov_b32 s2, s6
; CI-NEXT:    s_mov_b32 s3, s7
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    buffer_load_dword v0, off, s[0:3], 0 offset:400
; CI-NEXT:    buffer_load_dword v1, off, s[0:3], 0 offset:408
; CI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
; CI-NEXT:    v_mov_b32_e32 v2, 0x7b
; CI-NEXT:    v_mov_b32_e32 v3, 0x315
; CI-NEXT:    buffer_store_dword v2, off, s[0:3], 0 offset:12
; CI-NEXT:    buffer_store_dword v2, off, s[0:3], 0 offset:400
; CI-NEXT:    buffer_store_dword v3, off, s[0:3], 0 offset:408
; CI-NEXT:    s_waitcnt vmcnt(3)
; CI-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
; CI-NEXT:    v_add_i32_e32 v0, vcc, 0x7b, v0
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT:    s_endpgm
;
; GFX9-LABEL: reorder_global_offsets:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
; GFX9-NEXT:    v_mov_b32_e32 v0, 0
; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7b
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_dword v1, v0, s[0:1] offset:400
; GFX9-NEXT:    global_load_dword v2, v0, s[0:1] offset:408
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    global_store_dword v0, v3, s[0:1] offset:12
; GFX9-NEXT:    global_store_dword v0, v3, s[0:1] offset:400
; GFX9-NEXT:    v_mov_b32_e32 v3, 0x315
; GFX9-NEXT:    global_store_dword v0, v3, s[0:1] offset:408
; GFX9-NEXT:    s_waitcnt vmcnt(3)
; GFX9-NEXT:    v_add_u32_e32 v1, v2, v1
; GFX9-NEXT:    v_add_u32_e32 v1, 0x7b, v1
; GFX9-NEXT:    global_store_dword v0, v1, s[2:3]
; GFX9-NEXT:    s_endpgm
  %ptr1 = getelementptr inbounds i32, ptr addrspace(1) %ptr0, i32 3
  %ptr2 = getelementptr inbounds i32, ptr addrspace(1) %ptr0, i32 100
  %ptr3 = getelementptr inbounds i32, ptr addrspace(1) %ptr0, i32 102

  store i32 123, ptr addrspace(1) %ptr1, align 4
  %tmp1 = load i32, ptr addrspace(1) %ptr2, align 4
  %tmp2 = load i32, ptr addrspace(1) %ptr3, align 4
  store i32 123, ptr addrspace(1) %ptr2, align 4
  %tmp3 = load i32, ptr addrspace(1) %ptr1, align 4
  store i32 789, ptr addrspace(1) %ptr3, align 4

  %add.0 = add nsw i32 %tmp2, %tmp1
  %add.1 = add nsw i32 %add.0, %tmp3
  store i32 %add.1, ptr addrspace(1) %out, align 4
  ret void
}

define amdgpu_kernel void @reorder_global_offsets_addr64_soffset0(ptr addrspace(1) noalias nocapture %ptr.base) #0 {
; CI-LABEL: reorder_global_offsets_addr64_soffset0:
; CI:       ; %bb.0:
; CI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
; CI-NEXT:    s_mov_b32 s3, 0xf000
; CI-NEXT:    s_mov_b32 s2, 0
; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; CI-NEXT:    v_mov_b32_e32 v1, 0
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 offset:12
; CI-NEXT:    buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:28
; CI-NEXT:    buffer_load_dword v4, v[0:1], s[0:3], 0 addr64 offset:44
; CI-NEXT:    v_mov_b32_e32 v5, 0x315
; CI-NEXT:    v_mov_b32_e32 v6, 0x7b
; CI-NEXT:    buffer_store_dword v5, v[0:1], s[0:3], 0 addr64
; CI-NEXT:    buffer_store_dword v6, v[0:1], s[0:3], 0 addr64 offset:20
; CI-NEXT:    s_waitcnt vmcnt(3)
; CI-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:36
; CI-NEXT:    s_waitcnt vmcnt(3)
; CI-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:52
; CI-NEXT:    s_endpgm
;
; GFX9-LABEL: reorder_global_offsets_addr64_soffset0:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT:    v_mov_b32_e32 v4, 0x315
; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7b
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_dword v1, v0, s[0:1] offset:12
; GFX9-NEXT:    global_load_dword v2, v0, s[0:1] offset:28
; GFX9-NEXT:    global_load_dword v3, v0, s[0:1] offset:44
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    global_store_dword v0, v4, s[0:1]
; GFX9-NEXT:    global_store_dword v0, v5, s[0:1] offset:20
; GFX9-NEXT:    s_waitcnt vmcnt(3)
; GFX9-NEXT:    v_add_u32_e32 v1, v1, v2
; GFX9-NEXT:    global_store_dword v0, v1, s[0:1] offset:36
; GFX9-NEXT:    s_waitcnt vmcnt(3)
; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
; GFX9-NEXT:    global_store_dword v0, v1, s[0:1] offset:52
; GFX9-NEXT:    s_endpgm
  %id = call i32 @llvm.amdgcn.workitem.id.x()
  %id.ext = sext i32 %id to i64

  %ptr0 = getelementptr inbounds i32, ptr addrspace(1) %ptr.base, i64 %id.ext
  %ptr1 = getelementptr inbounds i32, ptr addrspace(1) %ptr0, i32 3
  %ptr2 = getelementptr inbounds i32, ptr addrspace(1) %ptr0, i32 5
  %ptr3 = getelementptr inbounds i32, ptr addrspace(1) %ptr0, i32 7
  %ptr4 = getelementptr inbounds i32, ptr addrspace(1) %ptr0, i32 9
  %ptr5 = getelementptr inbounds i32, ptr addrspace(1) %ptr0, i32 11
  %ptr6 = getelementptr inbounds i32, ptr addrspace(1) %ptr0, i32 13

  store i32 789, ptr addrspace(1) %ptr0, align 4
  %tmp1 = load i32, ptr addrspace(1) %ptr1, align 4
  store i32 123, ptr addrspace(1) %ptr2, align 4
  %tmp2 = load i32, ptr addrspace(1) %ptr3, align 4
  %add.0 = add nsw i32 %tmp1, %tmp2
  store i32 %add.0, ptr addrspace(1) %ptr4, align 4
  %tmp3 = load i32, ptr addrspace(1) %ptr5, align 4
  %add.1 = add nsw i32 %add.0, %tmp3
  store i32 %add.1, ptr addrspace(1) %ptr6, align 4
  ret void
}

define amdgpu_vs void @reorder_local_load_tbuffer_store_local_load(ptr addrspace(1) %out, i32 %a1, i32 %vaddr) #0 {
; CI-LABEL: reorder_local_load_tbuffer_store_local_load:
; CI:       ; %bb.0:
; CI-NEXT:    v_mov_b32_e32 v4, stored_lds_ptr@abs32@lo
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_b32 v4, v4
; CI-NEXT:    s_mov_b32 s2, 0
; CI-NEXT:    s_mov_b32 s3, 0xf000
; CI-NEXT:    s_mov_b32 s0, s2
; CI-NEXT:    s_mov_b32 s1, s2
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    ds_read2_b32 v[4:5], v4 offset0:1 offset1:2
; CI-NEXT:    v_add_i32_e32 v3, vcc, 32, v3
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    tbuffer_store_format_xyzw v[2:5], v3, s[0:3], 0 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_SNORM_OGL] idxen glc slc
; CI-NEXT:    s_nop 0
; CI-NEXT:    v_add_i32_e32 v2, vcc, v4, v5
; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; CI-NEXT:    s_endpgm
;
; GFX9-LABEL: reorder_local_load_tbuffer_store_local_load:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    v_mov_b32_e32 v4, stored_lds_ptr@abs32@lo
; GFX9-NEXT:    ds_read_b32 v4, v4
; GFX9-NEXT:    v_add_u32_e32 v3, 32, v3
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    ds_read2_b32 v[4:5], v4 offset0:1 offset1:2
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    tbuffer_store_format_xyzw v[2:5], v3, s[0:3], 0 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_RESERVED_6] idxen glc slc
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    v_add_u32_e32 v2, v4, v5
; GFX9-NEXT:    global_store_dword v[0:1], v2, off
; GFX9-NEXT:    s_endpgm
  %ptr0 = load ptr addrspace(3), ptr addrspace(3) @stored_lds_ptr, align 4

  %ptr1 = getelementptr inbounds i32, ptr addrspace(3) %ptr0, i32 1
  %ptr2 = getelementptr inbounds i32, ptr addrspace(3) %ptr0, i32 2

  %tmp1 = load i32, ptr addrspace(3) %ptr1, align 4

  %vdata = insertelement <4 x i32> poison, i32 %a1, i32 0
  %vaddr.add = add i32 %vaddr, 32
  call void @llvm.amdgcn.struct.ptr.tbuffer.store.v4i32(<4 x i32> %vdata, ptr addrspace(8) poison, i32 %vaddr.add, i32 0, i32 0, i32 228, i32 3)

  %tmp2 = load i32, ptr addrspace(3) %ptr2, align 4

  %add = add nsw i32 %tmp1, %tmp2
  store i32 %add, ptr addrspace(1) %out, align 4
  ret void
}

declare void @llvm.memcpy.p3.p0(ptr addrspace(3), ptr, i64, i1)
declare void @llvm.amdgcn.s.barrier() #1
declare i32 @llvm.amdgcn.workitem.id.x() #2
declare void @llvm.amdgcn.struct.ptr.tbuffer.store.v4i32(<4 x i32>, ptr addrspace(8), i32, i32, i32, i32 immarg, i32 immarg) #3

attributes #0 = { nounwind }
attributes #1 = { convergent nounwind willreturn }
attributes #2 = { nounwind readnone speculatable willreturn }
attributes #3 = { nounwind willreturn writeonly }