test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti < %s | FileCheck -check-prefix=GFX6 %s
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s

; Test end to end matching of addressing modes when MUBUF is used for
; global memory.

define amdgpu_ps void @mubuf_store_sgpr_ptr(i32 addrspace(1)* inreg %ptr) {
; GFX6-LABEL: mubuf_store_sgpr_ptr:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    v_mov_b32_e32 v0, 0
; GFX6-NEXT:    s_mov_b32 s2, -1
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_sgpr_ptr:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    v_mov_b32_e32 v0, 0
; GFX7-NEXT:    s_mov_b32 s2, -1
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT:    s_endpgm
  store i32 0, i32 addrspace(1)* %ptr
  ret void
}

define amdgpu_ps void @mubuf_store_sgpr_ptr_offset4095(i32 addrspace(1)* inreg %ptr) {
; GFX6-LABEL: mubuf_store_sgpr_ptr_offset4095:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    v_mov_b32_e32 v0, 0
; GFX6-NEXT:    s_mov_b32 s2, -1
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], s4
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_sgpr_ptr_offset4095:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    v_mov_b32_e32 v0, 0
; GFX7-NEXT:    s_mov_b32 s2, -1
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], s4
; GFX7-NEXT:    s_endpgm
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4095
  store i32 0, i32 addrspace(1)* %gep
  ret void
}

define amdgpu_ps void @mubuf_store_sgpr_ptr_offset4294967296(i32 addrspace(1)* inreg %ptr) {
; GFX6-LABEL: mubuf_store_sgpr_ptr_offset4294967296:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s4, 0
; GFX6-NEXT:    s_mov_b32 s5, 4
; GFX6-NEXT:    v_mov_b32_e32 v0, s4
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    v_mov_b32_e32 v2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b32 s2, s4
; GFX6-NEXT:    v_mov_b32_e32 v1, s5
; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_sgpr_ptr_offset4294967296:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s4, 0
; GFX7-NEXT:    s_mov_b32 s5, 4
; GFX7-NEXT:    v_mov_b32_e32 v0, s4
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    v_mov_b32_e32 v2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b32 s2, s4
; GFX7-NEXT:    v_mov_b32_e32 v1, s5
; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4294967296
  store i32 0, i32 addrspace(1)* %gep
  ret void
}

define amdgpu_ps void @mubuf_store_sgpr_ptr_offset4294967297(i32 addrspace(1)* inreg %ptr) {
; GFX6-LABEL: mubuf_store_sgpr_ptr_offset4294967297:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s4, 4
; GFX6-NEXT:    s_mov_b32 s5, s4
; GFX6-NEXT:    v_mov_b32_e32 v0, s4
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    v_mov_b32_e32 v2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    v_mov_b32_e32 v1, s5
; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_sgpr_ptr_offset4294967297:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s4, 4
; GFX7-NEXT:    s_mov_b32 s5, s4
; GFX7-NEXT:    v_mov_b32_e32 v0, s4
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    v_mov_b32_e32 v2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    v_mov_b32_e32 v1, s5
; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4294967297
  store i32 0, i32 addrspace(1)* %gep
  ret void
}

define amdgpu_ps void @mubuf_store_sgpr_ptr_offset4096(i32 addrspace(1)* inreg %ptr) {
; GFX6-LABEL: mubuf_store_sgpr_ptr_offset4096:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    v_mov_b32_e32 v0, 0
; GFX6-NEXT:    s_mov_b32 s2, -1
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_movk_i32 s4, 0x4000
; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], s4
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_sgpr_ptr_offset4096:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    v_mov_b32_e32 v0, 0
; GFX7-NEXT:    s_mov_b32 s2, -1
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_movk_i32 s4, 0x4000
; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], s4
; GFX7-NEXT:    s_endpgm
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4096
  store i32 0, i32 addrspace(1)* %gep
  ret void
}

define amdgpu_ps void @mubuf_store_vgpr_ptr_offset4095(i32 addrspace(1)* %ptr) {
; GFX6-LABEL: mubuf_store_vgpr_ptr_offset4095:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    v_mov_b32_e32 v2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], s4 addr64
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_vgpr_ptr_offset4095:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    v_mov_b32_e32 v2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b64 s[0:1], 0
; GFX7-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], s4 addr64
; GFX7-NEXT:    s_endpgm
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4095
  store i32 0, i32 addrspace(1)* %gep
  ret void
}

define amdgpu_ps void @mubuf_store_vgpr_ptr_offset4294967296(i32 addrspace(1)* %ptr) {
; GFX6-LABEL: mubuf_store_vgpr_ptr_offset4294967296:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, 0
; GFX6-NEXT:    s_mov_b32 s1, 4
; GFX6-NEXT:    v_mov_b32_e32 v2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b32 s2, s0
; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_vgpr_ptr_offset4294967296:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, 0
; GFX7-NEXT:    s_mov_b32 s1, 4
; GFX7-NEXT:    v_mov_b32_e32 v2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b32 s2, s0
; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4294967296
  store i32 0, i32 addrspace(1)* %gep
  ret void
}

define amdgpu_ps void @mubuf_store_vgpr_ptr_offset4294967297(i32 addrspace(1)* %ptr) {
; GFX6-LABEL: mubuf_store_vgpr_ptr_offset4294967297:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, 4
; GFX6-NEXT:    s_mov_b32 s1, s0
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    v_mov_b32_e32 v2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_vgpr_ptr_offset4294967297:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, 4
; GFX7-NEXT:    s_mov_b32 s1, s0
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    v_mov_b32_e32 v2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4294967297
  store i32 0, i32 addrspace(1)* %gep
  ret void
}

define amdgpu_ps void @mubuf_store_vgpr_ptr_offset4096(i32 addrspace(1)* %ptr) {
; GFX6-LABEL: mubuf_store_vgpr_ptr_offset4096:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    v_mov_b32_e32 v2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-NEXT:    s_movk_i32 s4, 0x4000
; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], s4 addr64
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_vgpr_ptr_offset4096:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    v_mov_b32_e32 v2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b64 s[0:1], 0
; GFX7-NEXT:    s_movk_i32 s4, 0x4000
; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], s4 addr64
; GFX7-NEXT:    s_endpgm
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4096
  store i32 0, i32 addrspace(1)* %gep
  ret void
}

define amdgpu_ps void @mubuf_store_sgpr_ptr_sgpr_offset(i32 addrspace(1)* inreg %ptr, i32 inreg %soffset) {
; GFX6-LABEL: mubuf_store_sgpr_ptr_sgpr_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    s_bfe_i64 s[2:3], s[4:5], 0x200000
; GFX6-NEXT:    s_lshl_b64 s[4:5], s[2:3], 2
; GFX6-NEXT:    v_mov_b32_e32 v0, s4
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    v_mov_b32_e32 v2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    v_mov_b32_e32 v1, s5
; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_sgpr_ptr_sgpr_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    s_bfe_i64 s[2:3], s[4:5], 0x200000
; GFX7-NEXT:    s_lshl_b64 s[4:5], s[2:3], 2
; GFX7-NEXT:    v_mov_b32_e32 v0, s4
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    v_mov_b32_e32 v2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    v_mov_b32_e32 v1, s5
; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 %soffset
  store i32 0, i32 addrspace(1)* %gep
  ret void
}

define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset(i32 addrspace(1)* %ptr, i32 inreg %soffset) {
; GFX6-LABEL: mubuf_store_vgpr_ptr_sgpr_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    v_mov_b32_e32 v2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_vgpr_ptr_sgpr_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    v_mov_b32_e32 v2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 %soffset
  store i32 0, i32 addrspace(1)* %gep
  ret void
}

define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset_offset256(i32 addrspace(1)* %ptr, i32 inreg %soffset) {
; GFX6-LABEL: mubuf_store_vgpr_ptr_sgpr_offset_offset256:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    v_mov_b32_e32 v2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:1024
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_vgpr_ptr_sgpr_offset_offset256:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    v_mov_b32_e32 v2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:1024
; GFX7-NEXT:    s_endpgm
  %gep0 = getelementptr i32, i32 addrspace(1)* %ptr, i32 %soffset
  %gep1 = getelementptr i32, i32 addrspace(1)* %gep0, i32 256
  store i32 0, i32 addrspace(1)* %gep1
  ret void
}

define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset256_offset(i32 addrspace(1)* %ptr, i32 inreg %soffset) {
; GFX6-LABEL: mubuf_store_vgpr_ptr_sgpr_offset256_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_movk_i32 s4, 0x400
; GFX6-NEXT:    s_mov_b32 s5, 0
; GFX6-NEXT:    v_mov_b32_e32 v2, s4
; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
; GFX6-NEXT:    v_mov_b32_e32 v3, s5
; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
; GFX6-NEXT:    v_mov_b32_e32 v2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b32 s2, s5
; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_vgpr_ptr_sgpr_offset256_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_movk_i32 s4, 0x400
; GFX7-NEXT:    s_mov_b32 s5, 0
; GFX7-NEXT:    v_mov_b32_e32 v2, s4
; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
; GFX7-NEXT:    v_mov_b32_e32 v3, s5
; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
; GFX7-NEXT:    v_mov_b32_e32 v2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b32 s2, s5
; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
  %gep0 = getelementptr i32, i32 addrspace(1)* %ptr, i32 256
  %gep1 = getelementptr i32, i32 addrspace(1)* %gep0, i32 %soffset
  store i32 0, i32 addrspace(1)* %gep1
  ret void
}

define amdgpu_ps void @mubuf_store_sgpr_ptr_vgpr_offset(i32 addrspace(1)* inreg %ptr, i32 %voffset) {
; GFX6-LABEL: mubuf_store_sgpr_ptr_vgpr_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    v_mov_b32_e32 v2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_sgpr_ptr_vgpr_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    v_mov_b32_e32 v2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 %voffset
  store i32 0, i32 addrspace(1)* %gep
  ret void
}

define amdgpu_ps void @mubuf_store_sgpr_ptr_vgpr_offset_offset4095(i32 addrspace(1)* inreg %ptr, i32 %voffset) {
; GFX6-LABEL: mubuf_store_sgpr_ptr_vgpr_offset_offset4095:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    v_mov_b32_e32 v2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], s4 addr64
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_sgpr_ptr_vgpr_offset_offset4095:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    v_mov_b32_e32 v2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], s4 addr64
; GFX7-NEXT:    s_endpgm
  %gep0 = getelementptr i32, i32 addrspace(1)* %ptr, i32 %voffset
  %gep1 = getelementptr i32, i32 addrspace(1)* %gep0, i32 4095
  store i32 0, i32 addrspace(1)* %gep1
  ret void
}
define amdgpu_ps void @mubuf_store_sgpr_ptr_offset4095_vgpr_offset(i32 addrspace(1)* inreg %ptr, i32 %voffset) {
; GFX6-LABEL: mubuf_store_sgpr_ptr_offset4095_vgpr_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT:    s_add_u32 s0, s2, 0x3ffc
; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX6-NEXT:    s_addc_u32 s1, s3, 0
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    v_mov_b32_e32 v2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX6-NEXT:    s_endpgm
;
; GFX7-LABEL: mubuf_store_sgpr_ptr_offset4095_vgpr_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT:    s_add_u32 s0, s2, 0x3ffc
; GFX7-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX7-NEXT:    s_addc_u32 s1, s3, 0
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    v_mov_b32_e32 v2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX7-NEXT:    s_endpgm
  %gep0 = getelementptr i32, i32 addrspace(1)* %ptr, i32 4095
  %gep1 = getelementptr i32, i32 addrspace(1)* %gep0, i32 %voffset
  store i32 0, i32 addrspace(1)* %gep1
  ret void
}

define amdgpu_ps float @mubuf_load_sgpr_ptr(float addrspace(1)* inreg %ptr) {
; GFX6-LABEL: mubuf_load_sgpr_ptr:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    s_mov_b32 s2, -1
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_sgpr_ptr:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    s_mov_b32 s2, -1
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %val = load volatile float, float addrspace(1)* %ptr
  ret float %val
}

define amdgpu_ps float @mubuf_load_sgpr_ptr_offset4095(float addrspace(1)* inreg %ptr) {
; GFX6-LABEL: mubuf_load_sgpr_ptr_offset4095:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    s_mov_b32 s2, -1
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], s4 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_sgpr_ptr_offset4095:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    s_mov_b32 s2, -1
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX7-NEXT:    buffer_load_dword v0, off, s[0:3], s4 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr float, float addrspace(1)* %ptr, i64 4095
  %val = load volatile float, float addrspace(1)* %gep
  ret float %val
}

define amdgpu_ps float @mubuf_load_sgpr_ptr_offset4294967296(float addrspace(1)* inreg %ptr) {
; GFX6-LABEL: mubuf_load_sgpr_ptr_offset4294967296:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s4, 0
; GFX6-NEXT:    s_mov_b32 s5, 4
; GFX6-NEXT:    v_mov_b32_e32 v0, s4
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b32 s2, s4
; GFX6-NEXT:    v_mov_b32_e32 v1, s5
; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_sgpr_ptr_offset4294967296:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s4, 0
; GFX7-NEXT:    s_mov_b32 s5, 4
; GFX7-NEXT:    v_mov_b32_e32 v0, s4
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b32 s2, s4
; GFX7-NEXT:    v_mov_b32_e32 v1, s5
; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr float, float addrspace(1)* %ptr, i64 4294967296
  %val = load volatile float, float addrspace(1)* %gep
  ret float %val
}

define amdgpu_ps float @mubuf_load_sgpr_ptr_offset4294967297(float addrspace(1)* inreg %ptr) {
; GFX6-LABEL: mubuf_load_sgpr_ptr_offset4294967297:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s4, 4
; GFX6-NEXT:    s_mov_b32 s5, s4
; GFX6-NEXT:    v_mov_b32_e32 v0, s4
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    v_mov_b32_e32 v1, s5
; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_sgpr_ptr_offset4294967297:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s4, 4
; GFX7-NEXT:    s_mov_b32 s5, s4
; GFX7-NEXT:    v_mov_b32_e32 v0, s4
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    v_mov_b32_e32 v1, s5
; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr float, float addrspace(1)* %ptr, i64 4294967297
  %val = load volatile float, float addrspace(1)* %gep
  ret float %val
}

define amdgpu_ps float @mubuf_load_sgpr_ptr_offset4096(float addrspace(1)* inreg %ptr) {
; GFX6-LABEL: mubuf_load_sgpr_ptr_offset4096:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    s_mov_b32 s2, -1
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_movk_i32 s4, 0x4000
; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], s4 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_sgpr_ptr_offset4096:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    s_mov_b32 s2, -1
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_movk_i32 s4, 0x4000
; GFX7-NEXT:    buffer_load_dword v0, off, s[0:3], s4 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr float, float addrspace(1)* %ptr, i64 4096
  %val = load volatile float, float addrspace(1)* %gep
  ret float %val
}

define amdgpu_ps float @mubuf_load_vgpr_ptr_offset4095(float addrspace(1)* %ptr) {
; GFX6-LABEL: mubuf_load_vgpr_ptr_offset4095:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], s4 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_vgpr_ptr_offset4095:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b64 s[0:1], 0
; GFX7-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], s4 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr float, float addrspace(1)* %ptr, i64 4095
  %val = load volatile float, float addrspace(1)* %gep
  ret float %val
}

define amdgpu_ps float @mubuf_load_vgpr_ptr_offset4294967296(float addrspace(1)* %ptr) {
; GFX6-LABEL: mubuf_load_vgpr_ptr_offset4294967296:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, 0
; GFX6-NEXT:    s_mov_b32 s1, 4
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b32 s2, s0
; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_vgpr_ptr_offset4294967296:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, 0
; GFX7-NEXT:    s_mov_b32 s1, 4
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b32 s2, s0
; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr float, float addrspace(1)* %ptr, i64 4294967296
  %val = load volatile float, float addrspace(1)* %gep
  ret float %val
}

define amdgpu_ps float @mubuf_load_vgpr_ptr_offset4294967297(float addrspace(1)* %ptr) {
; GFX6-LABEL: mubuf_load_vgpr_ptr_offset4294967297:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, 4
; GFX6-NEXT:    s_mov_b32 s1, s0
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_vgpr_ptr_offset4294967297:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, 4
; GFX7-NEXT:    s_mov_b32 s1, s0
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr float, float addrspace(1)* %ptr, i64 4294967297
  %val = load volatile float, float addrspace(1)* %gep
  ret float %val
}

define amdgpu_ps float @mubuf_load_vgpr_ptr_offset4096(float addrspace(1)* %ptr) {
; GFX6-LABEL: mubuf_load_vgpr_ptr_offset4096:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-NEXT:    s_movk_i32 s4, 0x4000
; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], s4 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_vgpr_ptr_offset4096:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b64 s[0:1], 0
; GFX7-NEXT:    s_movk_i32 s4, 0x4000
; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], s4 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr float, float addrspace(1)* %ptr, i64 4096
  %val = load volatile float, float addrspace(1)* %gep
  ret float %val
}

define amdgpu_ps float @mubuf_load_sgpr_ptr_sgpr_offset(float addrspace(1)* inreg %ptr, i32 inreg %soffset) {
; GFX6-LABEL: mubuf_load_sgpr_ptr_sgpr_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    s_bfe_i64 s[2:3], s[4:5], 0x200000
; GFX6-NEXT:    s_lshl_b64 s[4:5], s[2:3], 2
; GFX6-NEXT:    v_mov_b32_e32 v0, s4
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    v_mov_b32_e32 v1, s5
; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_sgpr_ptr_sgpr_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    s_bfe_i64 s[2:3], s[4:5], 0x200000
; GFX7-NEXT:    s_lshl_b64 s[4:5], s[2:3], 2
; GFX7-NEXT:    v_mov_b32_e32 v0, s4
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    v_mov_b32_e32 v1, s5
; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr float, float addrspace(1)* %ptr, i32 %soffset
  %val = load volatile float, float addrspace(1)* %gep
  ret float %val
}

define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset(float addrspace(1)* %ptr, i32 inreg %soffset) {
; GFX6-LABEL: mubuf_load_vgpr_ptr_sgpr_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_vgpr_ptr_sgpr_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr float, float addrspace(1)* %ptr, i32 %soffset
  %val = load volatile float, float addrspace(1)* %gep
  ret float %val
}

define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset_offset256(float addrspace(1)* %ptr, i32 inreg %soffset) {
; GFX6-LABEL: mubuf_load_vgpr_ptr_sgpr_offset_offset256:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:1024 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_vgpr_ptr_sgpr_offset_offset256:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:1024 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep0 = getelementptr float, float addrspace(1)* %ptr, i32 %soffset
  %gep1 = getelementptr float, float addrspace(1)* %gep0, i32 256
  %val = load volatile float, float addrspace(1)* %gep1
  ret float %val
}

define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset256_offset(float addrspace(1)* %ptr, i32 inreg %soffset) {
; GFX6-LABEL: mubuf_load_vgpr_ptr_sgpr_offset256_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_movk_i32 s4, 0x400
; GFX6-NEXT:    s_mov_b32 s5, 0
; GFX6-NEXT:    v_mov_b32_e32 v2, s4
; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
; GFX6-NEXT:    v_mov_b32_e32 v3, s5
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b32 s2, s5
; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_vgpr_ptr_sgpr_offset256_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_movk_i32 s4, 0x400
; GFX7-NEXT:    s_mov_b32 s5, 0
; GFX7-NEXT:    v_mov_b32_e32 v2, s4
; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
; GFX7-NEXT:    v_mov_b32_e32 v3, s5
; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b32 s2, s5
; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 256
  %gep1 = getelementptr float, float addrspace(1)* %gep0, i32 %soffset
  %val = load volatile float, float addrspace(1)* %gep1
  ret float %val
}

define amdgpu_ps float @mubuf_load_sgpr_ptr_vgpr_offset(float addrspace(1)* inreg %ptr, i32 %voffset) {
; GFX6-LABEL: mubuf_load_sgpr_ptr_vgpr_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_sgpr_ptr_vgpr_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr float, float addrspace(1)* %ptr, i32 %voffset
  %val = load volatile float, float addrspace(1)* %gep
  ret float %val
}

define amdgpu_ps float @mubuf_load_sgpr_ptr_vgpr_offset_offset4095(float addrspace(1)* inreg %ptr, i32 %voffset) {
; GFX6-LABEL: mubuf_load_sgpr_ptr_vgpr_offset_offset4095:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], s4 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_sgpr_ptr_vgpr_offset_offset4095:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], s4 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep0 = getelementptr float, float addrspace(1)* %ptr, i32 %voffset
  %gep1 = getelementptr float, float addrspace(1)* %gep0, i64 4095
  %val = load volatile float, float addrspace(1)* %gep1
  ret float %val
}
define amdgpu_ps float @mubuf_load_sgpr_ptr_offset4095_vgpr_offset(float addrspace(1)* inreg %ptr, i32 %voffset) {
; GFX6-LABEL: mubuf_load_sgpr_ptr_offset4095_vgpr_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT:    s_add_u32 s4, s2, 0x3ffc
; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX6-NEXT:    s_mov_b32 s6, 0
; GFX6-NEXT:    s_addc_u32 s5, s3, 0
; GFX6-NEXT:    s_mov_b32 s7, 0xf000
; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_load_sgpr_ptr_offset4095_vgpr_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT:    s_add_u32 s4, s2, 0x3ffc
; GFX7-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX7-NEXT:    s_mov_b32 s6, 0
; GFX7-NEXT:    s_addc_u32 s5, s3, 0
; GFX7-NEXT:    s_mov_b32 s7, 0xf000
; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    ; return to shader part epilog
  %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 4095
  %gep1 = getelementptr float, float addrspace(1)* %gep0, i32 %voffset
  %val = load volatile float, float addrspace(1)* %gep1
  ret float %val
}

define amdgpu_ps float @mubuf_atomicrmw_sgpr_ptr_offset4095(i32 addrspace(1)* inreg %ptr) {
; GFX6-LABEL: mubuf_atomicrmw_sgpr_ptr_offset4095:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    v_mov_b32_e32 v0, 2
; GFX6-NEXT:    s_mov_b32 s2, -1
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX6-NEXT:    buffer_atomic_add v0, off, s[0:3], s4 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_atomicrmw_sgpr_ptr_offset4095:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    v_mov_b32_e32 v0, 2
; GFX7-NEXT:    s_mov_b32 s2, -1
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT:    buffer_atomic_add v0, off, s[0:3], s4 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4095
  %result = atomicrmw add i32 addrspace(1)* %gep, i32 2 seq_cst
  %cast = bitcast i32 %result to float
  ret float %cast
}

define amdgpu_ps float @mubuf_atomicrmw_sgpr_ptr_offset4294967296(i32 addrspace(1)* inreg %ptr) {
; GFX6-LABEL: mubuf_atomicrmw_sgpr_ptr_offset4294967296:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s4, 0
; GFX6-NEXT:    s_mov_b32 s5, 4
; GFX6-NEXT:    v_mov_b32_e32 v1, s4
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    v_mov_b32_e32 v0, 2
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b32 s2, s4
; GFX6-NEXT:    v_mov_b32_e32 v2, s5
; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX6-NEXT:    buffer_atomic_add v0, v[1:2], s[0:3], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_atomicrmw_sgpr_ptr_offset4294967296:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s4, 0
; GFX7-NEXT:    s_mov_b32 s5, 4
; GFX7-NEXT:    v_mov_b32_e32 v1, s4
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    v_mov_b32_e32 v0, 2
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b32 s2, s4
; GFX7-NEXT:    v_mov_b32_e32 v2, s5
; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT:    buffer_atomic_add v0, v[1:2], s[0:3], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4294967296
  %result = atomicrmw add i32 addrspace(1)* %gep, i32 2 seq_cst
  %cast = bitcast i32 %result to float
  ret float %cast
}

define amdgpu_ps float @mubuf_atomicrmw_vgpr_ptr_offset4095(i32 addrspace(1)* %ptr) {
; GFX6-LABEL: mubuf_atomicrmw_vgpr_ptr_offset4095:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    v_mov_b32_e32 v2, 2
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX6-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], s4 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    v_mov_b32_e32 v0, v2
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_atomicrmw_vgpr_ptr_offset4095:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    v_mov_b32_e32 v2, 2
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b64 s[0:1], 0
; GFX7-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], s4 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1
; GFX7-NEXT:    v_mov_b32_e32 v0, v2
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4095
  %result = atomicrmw add i32 addrspace(1)* %gep, i32 2 seq_cst
  %cast = bitcast i32 %result to float
  ret float %cast
}

define amdgpu_ps float @mubuf_atomicrmw_vgpr_ptr_offset4294967296(i32 addrspace(1)* %ptr) {
; GFX6-LABEL: mubuf_atomicrmw_vgpr_ptr_offset4294967296:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, 0
; GFX6-NEXT:    s_mov_b32 s1, 4
; GFX6-NEXT:    v_mov_b32_e32 v2, 2
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b32 s2, s0
; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX6-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    v_mov_b32_e32 v0, v2
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_atomicrmw_vgpr_ptr_offset4294967296:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, 0
; GFX7-NEXT:    s_mov_b32 s1, 4
; GFX7-NEXT:    v_mov_b32_e32 v2, 2
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b32 s2, s0
; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1
; GFX7-NEXT:    v_mov_b32_e32 v0, v2
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4294967296
  %result = atomicrmw add i32 addrspace(1)* %gep, i32 2 seq_cst
  %cast = bitcast i32 %result to float
  ret float %cast
}

define amdgpu_ps float @mubuf_atomicrmw_sgpr_ptr_vgpr_offset(i32 addrspace(1)* inreg %ptr, i32 %voffset) {
; GFX6-LABEL: mubuf_atomicrmw_sgpr_ptr_vgpr_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    v_mov_b32_e32 v2, 2
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX6-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    v_mov_b32_e32 v0, v2
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_atomicrmw_sgpr_ptr_vgpr_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    v_mov_b32_e32 v2, 2
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT:    buffer_atomic_add v2, v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1
; GFX7-NEXT:    v_mov_b32_e32 v0, v2
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 %voffset
  %result = atomicrmw add i32 addrspace(1)* %gep, i32 2 seq_cst
  %cast = bitcast i32 %result to float
  ret float %cast
}

define amdgpu_ps float @mubuf_cmpxchg_sgpr_ptr_offset4095(i32 addrspace(1)* inreg %ptr, i32 %old, i32 %in) {
; GFX6-LABEL: mubuf_cmpxchg_sgpr_ptr_offset4095:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    v_mov_b32_e32 v2, v0
; GFX6-NEXT:    s_mov_b32 s2, -1
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX6-NEXT:    buffer_atomic_cmpswap v[1:2], off, s[0:3], s4 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    v_mov_b32_e32 v0, v1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_cmpxchg_sgpr_ptr_offset4095:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    v_mov_b32_e32 v2, v0
; GFX7-NEXT:    s_mov_b32 s2, -1
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT:    buffer_atomic_cmpswap v[1:2], off, s[0:3], s4 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1
; GFX7-NEXT:    v_mov_b32_e32 v0, v1
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4095
  %result.struct = cmpxchg i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst seq_cst
  %result = extractvalue { i32, i1 } %result.struct, 0
  %cast = bitcast i32 %result to float
  ret float %cast
}

define amdgpu_ps float @mubuf_cmpxchg_sgpr_ptr_offset4294967296(i32 addrspace(1)* inreg %ptr, i32 %old, i32 %in) {
; GFX6-LABEL: mubuf_cmpxchg_sgpr_ptr_offset4294967296:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s4, 0
; GFX6-NEXT:    s_mov_b32 s5, 4
; GFX6-NEXT:    v_mov_b32_e32 v3, s4
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    v_mov_b32_e32 v2, v0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b32 s2, s4
; GFX6-NEXT:    v_mov_b32_e32 v4, s5
; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX6-NEXT:    buffer_atomic_cmpswap v[1:2], v[3:4], s[0:3], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    v_mov_b32_e32 v0, v1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_cmpxchg_sgpr_ptr_offset4294967296:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s4, 0
; GFX7-NEXT:    s_mov_b32 s5, 4
; GFX7-NEXT:    v_mov_b32_e32 v3, s4
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    v_mov_b32_e32 v2, v0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b32 s2, s4
; GFX7-NEXT:    v_mov_b32_e32 v4, s5
; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT:    buffer_atomic_cmpswap v[1:2], v[3:4], s[0:3], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1
; GFX7-NEXT:    v_mov_b32_e32 v0, v1
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4294967296
  %result.struct = cmpxchg i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst seq_cst
  %result = extractvalue { i32, i1 } %result.struct, 0
  %cast = bitcast i32 %result to float
  ret float %cast
}

define amdgpu_ps float @mubuf_cmpxchg_vgpr_ptr_offset4095(i32 addrspace(1)* %ptr, i32 %old, i32 %in) {
; GFX6-LABEL: mubuf_cmpxchg_vgpr_ptr_offset4095:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    v_mov_b32_e32 v4, v2
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b64 s[0:1], 0
; GFX6-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX6-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[0:3], s4 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    v_mov_b32_e32 v0, v3
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_cmpxchg_vgpr_ptr_offset4095:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    v_mov_b32_e32 v4, v2
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b64 s[0:1], 0
; GFX7-NEXT:    s_movk_i32 s4, 0x3ffc
; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[0:3], s4 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1
; GFX7-NEXT:    v_mov_b32_e32 v0, v3
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4095
  %result.struct = cmpxchg i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst seq_cst
  %result = extractvalue { i32, i1 } %result.struct, 0
  %cast = bitcast i32 %result to float
  ret float %cast
}

define amdgpu_ps float @mubuf_cmpxchg_vgpr_ptr_offset4294967296(i32 addrspace(1)* %ptr, i32 %old, i32 %in) {
; GFX6-LABEL: mubuf_cmpxchg_vgpr_ptr_offset4294967296:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_mov_b32 s0, 0
; GFX6-NEXT:    v_mov_b32_e32 v4, v2
; GFX6-NEXT:    s_mov_b32 s1, 4
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_mov_b32 s2, s0
; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX6-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    v_mov_b32_e32 v0, v3
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_cmpxchg_vgpr_ptr_offset4294967296:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_mov_b32 s0, 0
; GFX7-NEXT:    v_mov_b32_e32 v4, v2
; GFX7-NEXT:    s_mov_b32 s1, 4
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_mov_b32 s2, s0
; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1
; GFX7-NEXT:    v_mov_b32_e32 v0, v3
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4294967296
  %result.struct = cmpxchg i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst seq_cst
  %result = extractvalue { i32, i1 } %result.struct, 0
  %cast = bitcast i32 %result to float
  ret float %cast
}

define amdgpu_ps float @mubuf_cmpxchg_sgpr_ptr_vgpr_offset(i32 addrspace(1)* inreg %ptr, i32 %voffset, i32 %old, i32 %in) {
; GFX6-LABEL: mubuf_cmpxchg_sgpr_ptr_vgpr_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    v_mov_b32_e32 v3, v1
; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX6-NEXT:    s_mov_b32 s0, s2
; GFX6-NEXT:    s_mov_b32 s1, s3
; GFX6-NEXT:    s_mov_b32 s2, 0
; GFX6-NEXT:    s_mov_b32 s3, 0xf000
; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX6-NEXT:    buffer_atomic_cmpswap v[2:3], v[0:1], s[0:3], 0 addr64 glc
; GFX6-NEXT:    s_waitcnt vmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    v_mov_b32_e32 v0, v2
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    ; return to shader part epilog
;
; GFX7-LABEL: mubuf_cmpxchg_sgpr_ptr_vgpr_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    v_mov_b32_e32 v3, v1
; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
; GFX7-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
; GFX7-NEXT:    s_mov_b32 s0, s2
; GFX7-NEXT:    s_mov_b32 s1, s3
; GFX7-NEXT:    s_mov_b32 s2, 0
; GFX7-NEXT:    s_mov_b32 s3, 0xf000
; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT:    buffer_atomic_cmpswap v[2:3], v[0:1], s[0:3], 0 addr64 glc
; GFX7-NEXT:    s_waitcnt vmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1
; GFX7-NEXT:    v_mov_b32_e32 v0, v2
; GFX7-NEXT:    ; return to shader part epilog
  %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 %voffset
  %result.struct = cmpxchg i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst seq_cst
  %result = extractvalue { i32, i1 } %result.struct, 0
  %cast = bitcast i32 %result to float
  ret float %cast
}