; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefixes=GCN1 %s
; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefixes=GCN2 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN3 %s

; ---------------------------------------------------------------------
; atomicrmw xchg
; ---------------------------------------------------------------------

define void @flat_atomic_xchg_i32_noret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_xchg_i32_noret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_atomic_swap v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i32_noret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_atomic_swap v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i32_noret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_swap v[0:1], v2
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw xchg ptr %ptr, i32 %in seq_cst
  ret void
}

define void @flat_atomic_xchg_i32_noret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_xchg_i32_noret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_swap v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i32_noret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_swap v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i32_noret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_swap v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw xchg ptr %gep, i32 %in seq_cst
  ret void
}

define i32 @flat_atomic_xchg_i32_ret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_xchg_i32_ret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i32_ret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i32_ret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xchg ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define i32 @flat_atomic_xchg_i32_ret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_xchg_i32_ret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i32_ret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i32_ret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_swap v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw xchg ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx void @flat_atomic_xchg_i32_noret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_i32_noret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    v_mov_b32_e32 v2, s6
; GCN1-NEXT:    flat_atomic_swap v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i32_noret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    v_mov_b32_e32 v2, s6
; GCN2-NEXT:    flat_atomic_swap v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i32_noret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    v_mov_b32_e32 v2, s6
; GCN3-NEXT:    flat_atomic_swap v[0:1], v2
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw xchg ptr %ptr, i32 %in seq_cst
  ret void
}

define amdgpu_gfx void @flat_atomic_xchg_i32_noret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_i32_noret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    v_mov_b32_e32 v2, s6
; GCN1-NEXT:    flat_atomic_swap v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i32_noret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    v_mov_b32_e32 v2, s6
; GCN2-NEXT:    flat_atomic_swap v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i32_noret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    v_mov_b32_e32 v2, s6
; GCN3-NEXT:    flat_atomic_swap v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw xchg ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_gfx i32 @flat_atomic_xchg_i32_ret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_i32_ret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    v_mov_b32_e32 v2, s6
; GCN1-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i32_ret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    v_mov_b32_e32 v2, s6
; GCN2-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i32_ret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    v_mov_b32_e32 v2, s6
; GCN3-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xchg ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx i32 @flat_atomic_xchg_i32_ret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_i32_ret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    v_mov_b32_e32 v2, s6
; GCN1-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i32_ret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    v_mov_b32_e32 v2, s6
; GCN2-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i32_ret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    v_mov_b32_e32 v2, s6
; GCN3-NEXT:    flat_atomic_swap v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw xchg ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define void @flat_atomic_xchg_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_xchg_i32_noret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_swap v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i32_noret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_swap v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i32_noret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_swap v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %tmp0 = atomicrmw xchg ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define i32 @flat_atomic_xchg_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_xchg_i32_ret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_i32_ret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_i32_ret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_swap v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %result = atomicrmw xchg ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

; ---------------------------------------------------------------------
; atomicrmw xchg f32
; ---------------------------------------------------------------------

define void @flat_atomic_xchg_f32_noret(ptr %ptr, float %in) {
; GCN1-LABEL: flat_atomic_xchg_f32_noret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_atomic_swap v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f32_noret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_atomic_swap v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f32_noret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_swap v[0:1], v2
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw xchg ptr %ptr, float %in seq_cst
  ret void
}

define void @flat_atomic_xchg_f32_noret_offset(ptr %out, float %in) {
; GCN1-LABEL: flat_atomic_xchg_f32_noret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_swap v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f32_noret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_swap v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f32_noret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_swap v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds float, ptr %out, i32 4
  %tmp0 = atomicrmw xchg ptr %gep, float %in seq_cst
  ret void
}

define float @flat_atomic_xchg_f32_ret(ptr %ptr, float %in) {
; GCN1-LABEL: flat_atomic_xchg_f32_ret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f32_ret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f32_ret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xchg ptr %ptr, float %in seq_cst
  ret float %result
}

define float @flat_atomic_xchg_f32_ret_offset(ptr %out, float %in) {
; GCN1-LABEL: flat_atomic_xchg_f32_ret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f32_ret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f32_ret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_swap v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds float, ptr %out, i32 4
  %result = atomicrmw xchg ptr %gep, float %in seq_cst
  ret float %result
}

define amdgpu_gfx void @flat_atomic_xchg_f32_noret_scalar(ptr inreg %ptr, float inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_f32_noret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    v_mov_b32_e32 v2, s6
; GCN1-NEXT:    flat_atomic_swap v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f32_noret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    v_mov_b32_e32 v2, s6
; GCN2-NEXT:    flat_atomic_swap v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f32_noret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    v_mov_b32_e32 v2, s6
; GCN3-NEXT:    flat_atomic_swap v[0:1], v2
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw xchg ptr %ptr, float %in seq_cst
  ret void
}

define amdgpu_gfx void @flat_atomic_xchg_f32_noret_offset_scalar(ptr inreg %out, float inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_f32_noret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    v_mov_b32_e32 v2, s6
; GCN1-NEXT:    flat_atomic_swap v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f32_noret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    v_mov_b32_e32 v2, s6
; GCN2-NEXT:    flat_atomic_swap v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f32_noret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    v_mov_b32_e32 v2, s6
; GCN3-NEXT:    flat_atomic_swap v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds float, ptr %out, i32 4
  %tmp0 = atomicrmw xchg ptr %gep, float %in seq_cst
  ret void
}

define amdgpu_gfx float @flat_atomic_xchg_f32_ret_scalar(ptr inreg %ptr, float inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_f32_ret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    v_mov_b32_e32 v2, s6
; GCN1-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f32_ret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    v_mov_b32_e32 v2, s6
; GCN2-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f32_ret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    v_mov_b32_e32 v2, s6
; GCN3-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xchg ptr %ptr, float %in seq_cst
  ret float %result
}

define amdgpu_gfx float @flat_atomic_xchg_f32_ret_offset_scalar(ptr inreg %out, float inreg %in) {
; GCN1-LABEL: flat_atomic_xchg_f32_ret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    v_mov_b32_e32 v2, s6
; GCN1-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f32_ret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    v_mov_b32_e32 v2, s6
; GCN2-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f32_ret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    v_mov_b32_e32 v2, s6
; GCN3-NEXT:    flat_atomic_swap v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds float, ptr %out, i32 4
  %result = atomicrmw xchg ptr %gep, float %in seq_cst
  ret float %result
}

define void @flat_atomic_xchg_f32_noret_offset__amdgpu_no_remote_memory(ptr %out, float %in) {
; GCN1-LABEL: flat_atomic_xchg_f32_noret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_swap v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f32_noret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_swap v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f32_noret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_swap v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds float, ptr %out, i64 4
  %tmp0 = atomicrmw xchg ptr %gep, float %in seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define float @flat_atomic_xchg_f32_ret_offset__amdgpu_no_remote_memory(ptr %out, float %in) {
; GCN1-LABEL: flat_atomic_xchg_f32_ret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xchg_f32_ret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_swap v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xchg_f32_ret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_swap v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds float, ptr %out, i64 4
  %result = atomicrmw xchg ptr %gep, float %in seq_cst, !amdgpu.no.remote.memory !0
  ret float %result
}

; ---------------------------------------------------------------------
; atomicrmw add
; ---------------------------------------------------------------------

define void @flat_atomic_add_i32_noret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_add_i32_noret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_atomic_add v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i32_noret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_atomic_add v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i32_noret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_add v[0:1], v2
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw add ptr %ptr, i32 %in seq_cst
  ret void
}

define void @flat_atomic_add_i32_noret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_add_i32_noret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_add v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i32_noret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_add v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i32_noret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_add v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw add ptr %gep, i32 %in seq_cst
  ret void
}

define i32 @flat_atomic_add_i32_ret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_add_i32_ret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i32_ret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i32_ret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw add ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define i32 @flat_atomic_add_i32_ret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_add_i32_ret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i32_ret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i32_ret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_add v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw add ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx void @flat_atomic_add_i32_noret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_add_i32_noret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    v_mov_b32_e32 v2, s6
; GCN1-NEXT:    flat_atomic_add v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i32_noret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    v_mov_b32_e32 v2, s6
; GCN2-NEXT:    flat_atomic_add v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i32_noret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    v_mov_b32_e32 v2, s6
; GCN3-NEXT:    flat_atomic_add v[0:1], v2
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw add ptr %ptr, i32 %in seq_cst
  ret void
}

define amdgpu_gfx void @flat_atomic_add_i32_noret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_add_i32_noret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    v_mov_b32_e32 v2, s6
; GCN1-NEXT:    flat_atomic_add v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i32_noret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    v_mov_b32_e32 v2, s6
; GCN2-NEXT:    flat_atomic_add v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i32_noret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    v_mov_b32_e32 v2, s6
; GCN3-NEXT:    flat_atomic_add v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw add ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_gfx i32 @flat_atomic_add_i32_ret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_add_i32_ret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    v_mov_b32_e32 v2, s6
; GCN1-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i32_ret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    v_mov_b32_e32 v2, s6
; GCN2-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i32_ret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    v_mov_b32_e32 v2, s6
; GCN3-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw add ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx i32 @flat_atomic_add_i32_ret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_add_i32_ret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    v_mov_b32_e32 v2, s6
; GCN1-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i32_ret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    v_mov_b32_e32 v2, s6
; GCN2-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i32_ret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    v_mov_b32_e32 v2, s6
; GCN3-NEXT:    flat_atomic_add v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw add ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define void @flat_atomic_add_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_add_i32_noret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_add v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i32_noret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_add v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i32_noret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_add v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %tmp0 = atomicrmw add ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define i32 @flat_atomic_add_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_add_i32_ret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_add_i32_ret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_add_i32_ret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_add v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %result = atomicrmw add ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

; ---------------------------------------------------------------------
; atomicrmw sub
; ---------------------------------------------------------------------

define void @flat_atomic_sub_i32_noret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_sub_i32_noret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB30_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_sub_i32_e32 v3, vcc, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB30_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i32_noret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB30_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_sub_u32_e32 v3, vcc, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB30_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i32_noret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB30_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_sub_u32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB30_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw sub ptr %ptr, i32 %in seq_cst
  ret void
}

define void @flat_atomic_sub_i32_noret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_sub_i32_noret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB31_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_sub_i32_e32 v3, vcc, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB31_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i32_noret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB31_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_sub_u32_e32 v3, vcc, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB31_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i32_noret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB31_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_sub_u32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB31_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw sub ptr %gep, i32 %in seq_cst
  ret void
}

define i32 @flat_atomic_sub_i32_ret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_sub_i32_ret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB32_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    v_sub_i32_e32 v3, vcc, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB32_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v0, v3
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i32_ret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB32_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    v_sub_u32_e32 v3, vcc, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB32_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v0, v3
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i32_ret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB32_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_sub_u32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB32_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw sub ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define i32 @flat_atomic_sub_i32_ret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_sub_i32_ret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v0, v[3:4]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB33_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v1, v0
; GCN1-NEXT:    v_sub_i32_e32 v0, vcc, v1, v2
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB33_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i32_ret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v0, v[3:4]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB33_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v1, v0
; GCN2-NEXT:    v_sub_u32_e32 v0, vcc, v1, v2
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB33_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i32_ret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB33_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_sub_u32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB33_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw sub ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx void @flat_atomic_sub_i32_noret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_sub_i32_noret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB34_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_subrev_i32_e32 v2, vcc, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB34_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i32_noret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB34_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_subrev_u32_e32 v2, vcc, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB34_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i32_noret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB34_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_subrev_u32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB34_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw sub ptr %ptr, i32 %in seq_cst
  ret void
}

define amdgpu_gfx void @flat_atomic_sub_i32_noret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_sub_i32_noret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB35_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_subrev_i32_e32 v2, vcc, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB35_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i32_noret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB35_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_subrev_u32_e32 v2, vcc, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB35_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i32_noret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB35_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_subrev_u32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB35_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw sub ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_gfx i32 @flat_atomic_sub_i32_ret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_sub_i32_ret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v0, v[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v1, s4
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:  .LBB36_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_subrev_i32_e32 v3, vcc, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB36_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i32_ret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v0, v[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v1, s4
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:  .LBB36_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_subrev_u32_e32 v3, vcc, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB36_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i32_ret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB36_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_subrev_u32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB36_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw sub ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx i32 @flat_atomic_sub_i32_ret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_sub_i32_ret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v1, s34
; GCN1-NEXT:    v_mov_b32_e32 v2, s35
; GCN1-NEXT:    flat_load_dword v0, v[1:2]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB37_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_subrev_i32_e32 v3, vcc, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB37_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i32_ret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v1, s34
; GCN2-NEXT:    v_mov_b32_e32 v2, s35
; GCN2-NEXT:    flat_load_dword v0, v[1:2]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB37_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_subrev_u32_e32 v3, vcc, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB37_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i32_ret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB37_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_subrev_u32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB37_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw sub ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define void @flat_atomic_sub_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_sub_i32_noret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_sub v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i32_noret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_sub v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i32_noret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_sub v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %tmp0 = atomicrmw sub ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define i32 @flat_atomic_sub_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_sub_i32_ret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i32_ret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i32_ret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_sub v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %result = atomicrmw sub ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

; ---------------------------------------------------------------------
; atomicrmw and
; ---------------------------------------------------------------------

define void @flat_atomic_and_i32_noret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_and_i32_noret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB40_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB40_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i32_noret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB40_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB40_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i32_noret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB40_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB40_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw and ptr %ptr, i32 %in seq_cst
  ret void
}

define void @flat_atomic_and_i32_noret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_and_i32_noret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB41_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB41_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i32_noret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB41_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB41_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i32_noret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB41_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB41_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw and ptr %gep, i32 %in seq_cst
  ret void
}

define i32 @flat_atomic_and_i32_ret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_and_i32_ret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB42_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB42_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v0, v3
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i32_ret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB42_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB42_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v0, v3
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i32_ret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB42_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB42_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw and ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define i32 @flat_atomic_and_i32_ret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_and_i32_ret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v0, v[3:4]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB43_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v1, v0
; GCN1-NEXT:    v_and_b32_e32 v0, v1, v2
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB43_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i32_ret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v0, v[3:4]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB43_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v1, v0
; GCN2-NEXT:    v_and_b32_e32 v0, v1, v2
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB43_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i32_ret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB43_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB43_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw and ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx void @flat_atomic_and_i32_noret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_and_i32_noret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB44_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB44_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i32_noret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB44_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB44_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i32_noret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB44_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_and_b32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB44_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw and ptr %ptr, i32 %in seq_cst
  ret void
}

define amdgpu_gfx void @flat_atomic_and_i32_noret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_and_i32_noret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB45_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB45_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i32_noret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB45_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB45_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i32_noret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB45_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_and_b32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB45_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw and ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_gfx i32 @flat_atomic_and_i32_ret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_and_i32_ret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v0, v[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v1, s4
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:  .LBB46_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_and_b32_e32 v3, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB46_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i32_ret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v0, v[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v1, s4
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:  .LBB46_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_and_b32_e32 v3, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB46_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i32_ret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB46_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_and_b32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB46_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw and ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx i32 @flat_atomic_and_i32_ret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_and_i32_ret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v1, s34
; GCN1-NEXT:    v_mov_b32_e32 v2, s35
; GCN1-NEXT:    flat_load_dword v0, v[1:2]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB47_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_and_b32_e32 v3, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB47_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i32_ret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v1, s34
; GCN2-NEXT:    v_mov_b32_e32 v2, s35
; GCN2-NEXT:    flat_load_dword v0, v[1:2]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB47_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_and_b32_e32 v3, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB47_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i32_ret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB47_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_and_b32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB47_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw and ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define void @flat_atomic_and_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_and_i32_noret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_and v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i32_noret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_and v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i32_noret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_and v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %tmp0 = atomicrmw and ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define i32 @flat_atomic_and_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_and_i32_ret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i32_ret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i32_ret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_and v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %result = atomicrmw and ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

; ---------------------------------------------------------------------
; atomicrmw nand
; ---------------------------------------------------------------------

define void @flat_atomic_nand_i32_noret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_nand_i32_noret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB50_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN1-NEXT:    v_not_b32_e32 v3, v3
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB50_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i32_noret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB50_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN2-NEXT:    v_not_b32_e32 v3, v3
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB50_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i32_noret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB50_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN3-NEXT:    v_not_b32_e32 v3, v3
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB50_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw nand ptr %ptr, i32 %in seq_cst
  ret void
}

define void @flat_atomic_nand_i32_noret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_nand_i32_noret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB51_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN1-NEXT:    v_not_b32_e32 v3, v3
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB51_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i32_noret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB51_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN2-NEXT:    v_not_b32_e32 v3, v3
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB51_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i32_noret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB51_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN3-NEXT:    v_not_b32_e32 v3, v3
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB51_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw nand ptr %gep, i32 %in seq_cst
  ret void
}

define i32 @flat_atomic_nand_i32_ret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_nand_i32_ret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB52_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN1-NEXT:    v_not_b32_e32 v3, v3
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB52_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v0, v3
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i32_ret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB52_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN2-NEXT:    v_not_b32_e32 v3, v3
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB52_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v0, v3
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i32_ret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB52_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN3-NEXT:    v_not_b32_e32 v3, v3
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB52_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw nand ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define i32 @flat_atomic_nand_i32_ret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_nand_i32_ret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v0, v[3:4]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB53_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v1, v0
; GCN1-NEXT:    v_and_b32_e32 v0, v1, v2
; GCN1-NEXT:    v_not_b32_e32 v0, v0
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB53_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i32_ret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v0, v[3:4]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB53_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v1, v0
; GCN2-NEXT:    v_and_b32_e32 v0, v1, v2
; GCN2-NEXT:    v_not_b32_e32 v0, v0
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB53_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i32_ret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB53_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN3-NEXT:    v_not_b32_e32 v3, v3
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB53_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw nand ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx void @flat_atomic_nand_i32_noret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_nand_i32_noret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB54_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v2, s6, v3
; GCN1-NEXT:    v_not_b32_e32 v2, v2
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB54_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i32_noret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB54_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v2, s6, v3
; GCN2-NEXT:    v_not_b32_e32 v2, v2
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB54_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i32_noret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB54_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_and_b32_e32 v2, s6, v3
; GCN3-NEXT:    v_not_b32_e32 v2, v2
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB54_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw nand ptr %ptr, i32 %in seq_cst
  ret void
}

define amdgpu_gfx void @flat_atomic_nand_i32_noret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_nand_i32_noret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB55_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v2, s6, v3
; GCN1-NEXT:    v_not_b32_e32 v2, v2
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB55_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i32_noret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB55_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v2, s6, v3
; GCN2-NEXT:    v_not_b32_e32 v2, v2
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB55_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i32_noret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB55_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_and_b32_e32 v2, s6, v3
; GCN3-NEXT:    v_not_b32_e32 v2, v2
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB55_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw nand ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_gfx i32 @flat_atomic_nand_i32_ret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_nand_i32_ret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v0, v[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v1, s4
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:  .LBB56_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_and_b32_e32 v0, s6, v4
; GCN1-NEXT:    v_not_b32_e32 v3, v0
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB56_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i32_ret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v0, v[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v1, s4
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:  .LBB56_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_and_b32_e32 v0, s6, v4
; GCN2-NEXT:    v_not_b32_e32 v3, v0
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB56_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i32_ret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB56_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_and_b32_e32 v0, s6, v4
; GCN3-NEXT:    v_not_b32_e32 v3, v0
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB56_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw nand ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx i32 @flat_atomic_nand_i32_ret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_nand_i32_ret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v1, s34
; GCN1-NEXT:    v_mov_b32_e32 v2, s35
; GCN1-NEXT:    flat_load_dword v0, v[1:2]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB57_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_and_b32_e32 v0, s6, v4
; GCN1-NEXT:    v_not_b32_e32 v3, v0
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB57_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i32_ret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v1, s34
; GCN2-NEXT:    v_mov_b32_e32 v2, s35
; GCN2-NEXT:    flat_load_dword v0, v[1:2]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB57_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_and_b32_e32 v0, s6, v4
; GCN2-NEXT:    v_not_b32_e32 v3, v0
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB57_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i32_ret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB57_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_and_b32_e32 v0, s6, v4
; GCN3-NEXT:    v_not_b32_e32 v3, v0
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB57_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw nand ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define void @flat_atomic_nand_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_nand_i32_noret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB58_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN1-NEXT:    v_not_b32_e32 v3, v3
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB58_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i32_noret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB58_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN2-NEXT:    v_not_b32_e32 v3, v3
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB58_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i32_noret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB58_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN3-NEXT:    v_not_b32_e32 v3, v3
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB58_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %tmp0 = atomicrmw nand ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define i32 @flat_atomic_nand_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_nand_i32_ret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v0, v[3:4]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB59_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v1, v0
; GCN1-NEXT:    v_and_b32_e32 v0, v1, v2
; GCN1-NEXT:    v_not_b32_e32 v0, v0
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB59_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i32_ret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v0, v[3:4]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB59_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v1, v0
; GCN2-NEXT:    v_and_b32_e32 v0, v1, v2
; GCN2-NEXT:    v_not_b32_e32 v0, v0
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB59_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i32_ret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB59_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_and_b32_e32 v3, v4, v2
; GCN3-NEXT:    v_not_b32_e32 v3, v3
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB59_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %result = atomicrmw nand ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

; ---------------------------------------------------------------------
; atomicrmw or
; ---------------------------------------------------------------------

define void @flat_atomic_or_i32_noret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_or_i32_noret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB60_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB60_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i32_noret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB60_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB60_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i32_noret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB60_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_or_b32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB60_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw or ptr %ptr, i32 %in seq_cst
  ret void
}

define void @flat_atomic_or_i32_noret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_or_i32_noret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB61_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB61_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i32_noret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB61_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB61_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i32_noret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB61_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_or_b32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB61_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw or ptr %gep, i32 %in seq_cst
  ret void
}

define i32 @flat_atomic_or_i32_ret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_or_i32_ret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB62_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    v_or_b32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB62_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v0, v3
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i32_ret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB62_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    v_or_b32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB62_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v0, v3
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i32_ret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB62_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_or_b32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB62_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw or ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define i32 @flat_atomic_or_i32_ret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_or_i32_ret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v0, v[3:4]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB63_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v1, v0
; GCN1-NEXT:    v_or_b32_e32 v0, v1, v2
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB63_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i32_ret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v0, v[3:4]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB63_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v1, v0
; GCN2-NEXT:    v_or_b32_e32 v0, v1, v2
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB63_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i32_ret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB63_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_or_b32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB63_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw or ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx void @flat_atomic_or_i32_noret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_or_i32_noret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB64_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB64_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i32_noret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB64_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB64_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i32_noret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB64_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_or_b32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB64_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw or ptr %ptr, i32 %in seq_cst
  ret void
}

define amdgpu_gfx void @flat_atomic_or_i32_noret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_or_i32_noret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB65_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_or_b32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB65_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i32_noret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB65_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_or_b32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB65_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i32_noret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB65_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_or_b32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB65_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw or ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_gfx i32 @flat_atomic_or_i32_ret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_or_i32_ret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v0, v[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v1, s4
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:  .LBB66_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_or_b32_e32 v3, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB66_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i32_ret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v0, v[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v1, s4
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:  .LBB66_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_or_b32_e32 v3, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB66_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i32_ret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB66_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_or_b32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB66_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw or ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx i32 @flat_atomic_or_i32_ret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_or_i32_ret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v1, s34
; GCN1-NEXT:    v_mov_b32_e32 v2, s35
; GCN1-NEXT:    flat_load_dword v0, v[1:2]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB67_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_or_b32_e32 v3, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB67_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i32_ret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v1, s34
; GCN2-NEXT:    v_mov_b32_e32 v2, s35
; GCN2-NEXT:    flat_load_dword v0, v[1:2]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB67_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_or_b32_e32 v3, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB67_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i32_ret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB67_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_or_b32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB67_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw or ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define void @flat_atomic_or_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_or_i32_noret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_or v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i32_noret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_or v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i32_noret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_or v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %tmp0 = atomicrmw or ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define i32 @flat_atomic_or_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_or_i32_ret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i32_ret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i32_ret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_or v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %result = atomicrmw or ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

; ---------------------------------------------------------------------
; atomicrmw xor
; ---------------------------------------------------------------------

define void @flat_atomic_xor_i32_noret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_xor_i32_noret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB70_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB70_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i32_noret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB70_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB70_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i32_noret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB70_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_xor_b32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB70_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw xor ptr %ptr, i32 %in seq_cst
  ret void
}

define void @flat_atomic_xor_i32_noret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_xor_i32_noret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB71_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB71_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i32_noret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB71_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB71_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i32_noret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB71_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_xor_b32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB71_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw xor ptr %gep, i32 %in seq_cst
  ret void
}

define i32 @flat_atomic_xor_i32_ret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_xor_i32_ret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB72_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    v_xor_b32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB72_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v0, v3
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i32_ret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB72_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    v_xor_b32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB72_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v0, v3
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i32_ret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB72_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_xor_b32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB72_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xor ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define i32 @flat_atomic_xor_i32_ret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_xor_i32_ret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v0, v[3:4]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB73_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v1, v0
; GCN1-NEXT:    v_xor_b32_e32 v0, v1, v2
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB73_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i32_ret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v0, v[3:4]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB73_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v1, v0
; GCN2-NEXT:    v_xor_b32_e32 v0, v1, v2
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB73_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i32_ret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB73_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_xor_b32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB73_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw xor ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx void @flat_atomic_xor_i32_noret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_xor_i32_noret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB74_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB74_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i32_noret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB74_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB74_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i32_noret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB74_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_xor_b32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB74_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw xor ptr %ptr, i32 %in seq_cst
  ret void
}

define amdgpu_gfx void @flat_atomic_xor_i32_noret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_xor_i32_noret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB75_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_xor_b32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB75_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i32_noret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB75_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_xor_b32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB75_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i32_noret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB75_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_xor_b32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB75_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw xor ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_gfx i32 @flat_atomic_xor_i32_ret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_xor_i32_ret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v0, v[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v1, s4
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:  .LBB76_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_xor_b32_e32 v3, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB76_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i32_ret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v0, v[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v1, s4
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:  .LBB76_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_xor_b32_e32 v3, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB76_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i32_ret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB76_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_xor_b32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB76_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw xor ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx i32 @flat_atomic_xor_i32_ret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_xor_i32_ret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v1, s34
; GCN1-NEXT:    v_mov_b32_e32 v2, s35
; GCN1-NEXT:    flat_load_dword v0, v[1:2]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB77_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_xor_b32_e32 v3, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB77_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i32_ret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v1, s34
; GCN2-NEXT:    v_mov_b32_e32 v2, s35
; GCN2-NEXT:    flat_load_dword v0, v[1:2]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB77_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_xor_b32_e32 v3, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB77_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i32_ret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB77_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_xor_b32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB77_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw xor ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define void @flat_xor_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_xor_i32_noret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_xor v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_xor_i32_noret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_xor v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_xor_i32_noret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_xor v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %tmp0 = atomicrmw xor ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define i32 @flat_atomic_xor_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_xor_i32_ret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i32_ret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i32_ret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_xor v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %result = atomicrmw xor ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

; ---------------------------------------------------------------------
; atomicrmw max
; ---------------------------------------------------------------------

define void @flat_atomic_max_i32_noret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_max_i32_noret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB80_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_max_i32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB80_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i32_noret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB80_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_max_i32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB80_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i32_noret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB80_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_max_i32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB80_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw max ptr %ptr, i32 %in seq_cst
  ret void
}

define void @flat_atomic_max_i32_noret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_max_i32_noret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB81_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_max_i32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB81_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i32_noret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB81_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_max_i32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB81_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i32_noret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB81_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_max_i32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB81_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw max ptr %gep, i32 %in seq_cst
  ret void
}

define i32 @flat_atomic_max_i32_ret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_max_i32_ret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB82_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    v_max_i32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB82_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v0, v3
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i32_ret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB82_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    v_max_i32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB82_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v0, v3
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i32_ret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB82_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_max_i32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB82_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw max ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define i32 @flat_atomic_max_i32_ret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_max_i32_ret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v0, v[3:4]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB83_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v1, v0
; GCN1-NEXT:    v_max_i32_e32 v0, v1, v2
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB83_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i32_ret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v0, v[3:4]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB83_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v1, v0
; GCN2-NEXT:    v_max_i32_e32 v0, v1, v2
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB83_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i32_ret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB83_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_max_i32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB83_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw max ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx void @flat_atomic_max_i32_noret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_max_i32_noret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB84_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_max_i32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB84_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i32_noret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB84_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_max_i32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB84_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i32_noret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB84_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_max_i32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB84_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw max ptr %ptr, i32 %in seq_cst
  ret void
}

define amdgpu_gfx void @flat_atomic_max_i32_noret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_max_i32_noret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB85_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_max_i32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB85_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i32_noret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB85_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_max_i32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB85_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i32_noret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB85_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_max_i32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB85_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw max ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_gfx i32 @flat_atomic_max_i32_ret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_max_i32_ret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v0, v[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v1, s4
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:  .LBB86_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_max_i32_e32 v3, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB86_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i32_ret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v0, v[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v1, s4
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:  .LBB86_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_max_i32_e32 v3, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB86_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i32_ret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB86_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_max_i32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB86_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw max ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx i32 @flat_atomic_max_i32_ret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_max_i32_ret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v1, s34
; GCN1-NEXT:    v_mov_b32_e32 v2, s35
; GCN1-NEXT:    flat_load_dword v0, v[1:2]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB87_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_max_i32_e32 v3, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB87_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i32_ret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v1, s34
; GCN2-NEXT:    v_mov_b32_e32 v2, s35
; GCN2-NEXT:    flat_load_dword v0, v[1:2]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB87_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_max_i32_e32 v3, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB87_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i32_ret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB87_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_max_i32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB87_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw max ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr %out, i32 %in, i32 %index) {
; GCN1-LABEL: atomic_max_i32_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_ashr_i32 s5, s3, 31
; GCN1-NEXT:    s_mov_b32 s4, s3
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 16
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[0:1], 0
; GCN1-NEXT:  .LBB88_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_max_i32_e32 v2, s2, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT:    s_cbranch_execnz .LBB88_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i32_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_ashr_i32 s5, s3, 31
; GCN2-NEXT:    s_mov_b32 s4, s3
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 16
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[0:1], 0
; GCN2-NEXT:  .LBB88_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_max_i32_e32 v2, s2, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT:    s_cbranch_execnz .LBB88_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_endpgm
;
; GCN3-LABEL: atomic_max_i32_addr64_offset:
; GCN3:       ; %bb.0: ; %entry
; GCN3-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
; GCN3-NEXT:    s_ashr_i32 s5, s3, 31
; GCN3-NEXT:    s_mov_b32 s4, s3
; GCN3-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN3-NEXT:    s_add_u32 s0, s0, s4
; GCN3-NEXT:    s_addc_u32 s1, s1, s5
; GCN3-NEXT:    v_mov_b32_e32 v0, s0
; GCN3-NEXT:    v_mov_b32_e32 v1, s1
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[0:1], 0
; GCN3-NEXT:  .LBB88_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_max_i32_e32 v2, s2, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT:    s_cbranch_execnz .LBB88_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i32, ptr %out, i32 %index
  %gep = getelementptr inbounds i32, ptr %ptr, i32 4
  %tmp0 = atomicrmw max ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i32 %index) {
; GCN1-LABEL: atomic_max_i32_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_ashr_i32 s5, s7, 31
; GCN1-NEXT:    s_mov_b32 s4, s7
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 16
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dword v2, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[0:1], 0
; GCN1-NEXT:  .LBB89_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    v_max_i32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT:    s_cbranch_execnz .LBB89_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s3
; GCN1-NEXT:    flat_store_dword v[0:1], v2
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i32_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_ashr_i32 s5, s7, 31
; GCN2-NEXT:    s_mov_b32 s4, s7
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 16
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dword v2, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[0:1], 0
; GCN2-NEXT:  .LBB89_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    v_max_i32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT:    s_cbranch_execnz .LBB89_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s3
; GCN2-NEXT:    flat_store_dword v[0:1], v2
; GCN2-NEXT:    s_endpgm
;
; GCN3-LABEL: atomic_max_i32_ret_addr64_offset:
; GCN3:       ; %bb.0: ; %entry
; GCN3-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN3-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
; GCN3-NEXT:    s_ashr_i32 s5, s7, 31
; GCN3-NEXT:    s_mov_b32 s4, s7
; GCN3-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN3-NEXT:    s_add_u32 s0, s0, s4
; GCN3-NEXT:    s_addc_u32 s1, s1, s5
; GCN3-NEXT:    v_mov_b32_e32 v0, s0
; GCN3-NEXT:    v_mov_b32_e32 v1, s1
; GCN3-NEXT:    flat_load_dword v2, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[0:1], 0
; GCN3-NEXT:  .LBB89_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    v_max_i32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT:    s_cbranch_execnz .LBB89_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v0, s2
; GCN3-NEXT:    v_mov_b32_e32 v1, s3
; GCN3-NEXT:    flat_store_dword v[0:1], v2
; GCN3-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i32, ptr %out, i32 %index
  %gep = getelementptr inbounds i32, ptr %ptr, i32 4
  %tmp0 = atomicrmw max ptr %gep, i32 %in seq_cst
  store i32 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_max_i32_addr64(ptr %out, i32 %in, i32 %index) {
; GCN1-LABEL: atomic_max_i32_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_ashr_i32 s5, s3, 31
; GCN1-NEXT:    s_mov_b32 s4, s3
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[0:1], 0
; GCN1-NEXT:  .LBB90_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_max_i32_e32 v2, s2, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT:    s_cbranch_execnz .LBB90_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i32_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_ashr_i32 s5, s3, 31
; GCN2-NEXT:    s_mov_b32 s4, s3
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[0:1], 0
; GCN2-NEXT:  .LBB90_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_max_i32_e32 v2, s2, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT:    s_cbranch_execnz .LBB90_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_endpgm
;
; GCN3-LABEL: atomic_max_i32_addr64:
; GCN3:       ; %bb.0: ; %entry
; GCN3-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
; GCN3-NEXT:    s_ashr_i32 s5, s3, 31
; GCN3-NEXT:    s_mov_b32 s4, s3
; GCN3-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN3-NEXT:    s_add_u32 s0, s0, s4
; GCN3-NEXT:    s_addc_u32 s1, s1, s5
; GCN3-NEXT:    v_mov_b32_e32 v0, s0
; GCN3-NEXT:    v_mov_b32_e32 v1, s1
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[0:1], 0
; GCN3-NEXT:  .LBB90_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_max_i32_e32 v2, s2, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT:    s_cbranch_execnz .LBB90_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i32, ptr %out, i32 %index
  %tmp0 = atomicrmw max ptr %ptr, i32 %in seq_cst
  ret void
}

define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i32 %index) {
; GCN1-LABEL: atomic_max_i32_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_ashr_i32 s5, s7, 31
; GCN1-NEXT:    s_mov_b32 s4, s7
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dword v2, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[0:1], 0
; GCN1-NEXT:  .LBB91_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    v_max_i32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT:    s_cbranch_execnz .LBB91_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s3
; GCN1-NEXT:    flat_store_dword v[0:1], v2
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_max_i32_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_ashr_i32 s5, s7, 31
; GCN2-NEXT:    s_mov_b32 s4, s7
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dword v2, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[0:1], 0
; GCN2-NEXT:  .LBB91_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    v_max_i32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT:    s_cbranch_execnz .LBB91_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s3
; GCN2-NEXT:    flat_store_dword v[0:1], v2
; GCN2-NEXT:    s_endpgm
;
; GCN3-LABEL: atomic_max_i32_ret_addr64:
; GCN3:       ; %bb.0: ; %entry
; GCN3-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN3-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
; GCN3-NEXT:    s_ashr_i32 s5, s7, 31
; GCN3-NEXT:    s_mov_b32 s4, s7
; GCN3-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN3-NEXT:    s_add_u32 s0, s0, s4
; GCN3-NEXT:    s_addc_u32 s1, s1, s5
; GCN3-NEXT:    v_mov_b32_e32 v0, s0
; GCN3-NEXT:    v_mov_b32_e32 v1, s1
; GCN3-NEXT:    flat_load_dword v2, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[0:1], 0
; GCN3-NEXT:  .LBB91_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    v_max_i32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT:    s_cbranch_execnz .LBB91_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v0, s2
; GCN3-NEXT:    v_mov_b32_e32 v1, s3
; GCN3-NEXT:    flat_store_dword v[0:1], v2
; GCN3-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i32, ptr %out, i32 %index
  %tmp0 = atomicrmw max ptr %ptr, i32 %in seq_cst
  store i32 %tmp0, ptr %out2
  ret void
}

define void @flat_max_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_max_i32_noret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_smax v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_max_i32_noret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_smax v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_max_i32_noret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_smax v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %tmp0 = atomicrmw max ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define i32 @flat_atomic_max_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_max_i32_ret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_smax v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i32_ret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_smax v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i32_ret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_smax v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %result = atomicrmw max ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

; ---------------------------------------------------------------------
; atomicrmw umax
; ---------------------------------------------------------------------

define void @flat_atomic_umax_i32_noret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_umax_i32_noret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB94_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_max_u32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB94_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i32_noret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB94_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_max_u32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB94_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i32_noret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB94_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_max_u32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB94_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw umax ptr %ptr, i32 %in seq_cst
  ret void
}

define void @flat_atomic_umax_i32_noret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_umax_i32_noret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB95_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_max_u32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB95_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i32_noret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB95_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_max_u32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB95_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i32_noret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB95_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_max_u32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB95_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw umax ptr %gep, i32 %in seq_cst
  ret void
}

define i32 @flat_atomic_umax_i32_ret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_umax_i32_ret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB96_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    v_max_u32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB96_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v0, v3
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i32_ret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB96_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    v_max_u32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB96_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v0, v3
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i32_ret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB96_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_max_u32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB96_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw umax ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define i32 @flat_atomic_umax_i32_ret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_umax_i32_ret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v0, v[3:4]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB97_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v1, v0
; GCN1-NEXT:    v_max_u32_e32 v0, v1, v2
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB97_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i32_ret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v0, v[3:4]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB97_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v1, v0
; GCN2-NEXT:    v_max_u32_e32 v0, v1, v2
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB97_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i32_ret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB97_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_max_u32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB97_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw umax ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx void @flat_atomic_umax_i32_noret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_umax_i32_noret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB98_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_max_u32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB98_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i32_noret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB98_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_max_u32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB98_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i32_noret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB98_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_max_u32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB98_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw umax ptr %ptr, i32 %in seq_cst
  ret void
}

define amdgpu_gfx void @flat_atomic_umax_i32_noret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_umax_i32_noret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB99_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_max_u32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB99_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i32_noret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB99_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_max_u32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB99_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i32_noret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB99_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_max_u32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB99_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw umax ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_gfx i32 @flat_atomic_umax_i32_ret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_umax_i32_ret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v0, v[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v1, s4
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:  .LBB100_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_max_u32_e32 v3, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB100_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i32_ret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v0, v[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v1, s4
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:  .LBB100_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_max_u32_e32 v3, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB100_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i32_ret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB100_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_max_u32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB100_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw umax ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx i32 @flat_atomic_umax_i32_ret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_umax_i32_ret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v1, s34
; GCN1-NEXT:    v_mov_b32_e32 v2, s35
; GCN1-NEXT:    flat_load_dword v0, v[1:2]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB101_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_max_u32_e32 v3, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB101_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i32_ret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v1, s34
; GCN2-NEXT:    v_mov_b32_e32 v2, s35
; GCN2-NEXT:    flat_load_dword v0, v[1:2]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB101_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_max_u32_e32 v3, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB101_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i32_ret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB101_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_max_u32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB101_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw umax ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr %out, i32 %in, i32 %index) {
; GCN1-LABEL: atomic_umax_i32_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_ashr_i32 s5, s3, 31
; GCN1-NEXT:    s_mov_b32 s4, s3
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 16
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[0:1], 0
; GCN1-NEXT:  .LBB102_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_max_u32_e32 v2, s2, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT:    s_cbranch_execnz .LBB102_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i32_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_ashr_i32 s5, s3, 31
; GCN2-NEXT:    s_mov_b32 s4, s3
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 16
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[0:1], 0
; GCN2-NEXT:  .LBB102_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_max_u32_e32 v2, s2, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT:    s_cbranch_execnz .LBB102_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_endpgm
;
; GCN3-LABEL: atomic_umax_i32_addr64_offset:
; GCN3:       ; %bb.0: ; %entry
; GCN3-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
; GCN3-NEXT:    s_ashr_i32 s5, s3, 31
; GCN3-NEXT:    s_mov_b32 s4, s3
; GCN3-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN3-NEXT:    s_add_u32 s0, s0, s4
; GCN3-NEXT:    s_addc_u32 s1, s1, s5
; GCN3-NEXT:    v_mov_b32_e32 v0, s0
; GCN3-NEXT:    v_mov_b32_e32 v1, s1
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[0:1], 0
; GCN3-NEXT:  .LBB102_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_max_u32_e32 v2, s2, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT:    s_cbranch_execnz .LBB102_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i32, ptr %out, i32 %index
  %gep = getelementptr inbounds i32, ptr %ptr, i32 4
  %tmp0 = atomicrmw umax ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i32 %index) {
; GCN1-LABEL: atomic_umax_i32_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_ashr_i32 s5, s7, 31
; GCN1-NEXT:    s_mov_b32 s4, s7
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 16
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dword v2, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[0:1], 0
; GCN1-NEXT:  .LBB103_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    v_max_u32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT:    s_cbranch_execnz .LBB103_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s3
; GCN1-NEXT:    flat_store_dword v[0:1], v2
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i32_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_ashr_i32 s5, s7, 31
; GCN2-NEXT:    s_mov_b32 s4, s7
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 16
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dword v2, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[0:1], 0
; GCN2-NEXT:  .LBB103_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    v_max_u32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT:    s_cbranch_execnz .LBB103_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s3
; GCN2-NEXT:    flat_store_dword v[0:1], v2
; GCN2-NEXT:    s_endpgm
;
; GCN3-LABEL: atomic_umax_i32_ret_addr64_offset:
; GCN3:       ; %bb.0: ; %entry
; GCN3-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN3-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
; GCN3-NEXT:    s_ashr_i32 s5, s7, 31
; GCN3-NEXT:    s_mov_b32 s4, s7
; GCN3-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN3-NEXT:    s_add_u32 s0, s0, s4
; GCN3-NEXT:    s_addc_u32 s1, s1, s5
; GCN3-NEXT:    v_mov_b32_e32 v0, s0
; GCN3-NEXT:    v_mov_b32_e32 v1, s1
; GCN3-NEXT:    flat_load_dword v2, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[0:1], 0
; GCN3-NEXT:  .LBB103_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    v_max_u32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT:    s_cbranch_execnz .LBB103_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v0, s2
; GCN3-NEXT:    v_mov_b32_e32 v1, s3
; GCN3-NEXT:    flat_store_dword v[0:1], v2
; GCN3-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i32, ptr %out, i32 %index
  %gep = getelementptr inbounds i32, ptr %ptr, i32 4
  %tmp0 = atomicrmw umax ptr %gep, i32 %in seq_cst
  store i32 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i32 %index) {
; GCN1-LABEL: atomic_umax_i32_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_ashr_i32 s5, s7, 31
; GCN1-NEXT:    s_mov_b32 s4, s7
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dword v2, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[0:1], 0
; GCN1-NEXT:  .LBB104_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    v_max_u32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT:    s_cbranch_execnz .LBB104_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s3
; GCN1-NEXT:    flat_store_dword v[0:1], v2
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_umax_i32_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_ashr_i32 s5, s7, 31
; GCN2-NEXT:    s_mov_b32 s4, s7
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dword v2, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[0:1], 0
; GCN2-NEXT:  .LBB104_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    v_max_u32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT:    s_cbranch_execnz .LBB104_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s3
; GCN2-NEXT:    flat_store_dword v[0:1], v2
; GCN2-NEXT:    s_endpgm
;
; GCN3-LABEL: atomic_umax_i32_ret_addr64:
; GCN3:       ; %bb.0: ; %entry
; GCN3-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN3-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
; GCN3-NEXT:    s_ashr_i32 s5, s7, 31
; GCN3-NEXT:    s_mov_b32 s4, s7
; GCN3-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN3-NEXT:    s_add_u32 s0, s0, s4
; GCN3-NEXT:    s_addc_u32 s1, s1, s5
; GCN3-NEXT:    v_mov_b32_e32 v0, s0
; GCN3-NEXT:    v_mov_b32_e32 v1, s1
; GCN3-NEXT:    flat_load_dword v2, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[0:1], 0
; GCN3-NEXT:  .LBB104_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    v_max_u32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT:    s_cbranch_execnz .LBB104_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v0, s2
; GCN3-NEXT:    v_mov_b32_e32 v1, s3
; GCN3-NEXT:    flat_store_dword v[0:1], v2
; GCN3-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i32, ptr %out, i32 %index
  %tmp0 = atomicrmw umax ptr %ptr, i32 %in seq_cst
  store i32 %tmp0, ptr %out2
  ret void
}

define void @flat_umax_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_umax_i32_noret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_umax v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_umax_i32_noret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_umax v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_umax_i32_noret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_umax v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %tmp0 = atomicrmw umax ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define i32 @flat_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_umax v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_umax v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_umax v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %result = atomicrmw umax ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

; ---------------------------------------------------------------------
; atomicrmw umin
; ---------------------------------------------------------------------

define void @flat_atomic_umin_i32_noret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_umin_i32_noret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB107_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_min_u32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB107_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i32_noret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB107_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_min_u32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB107_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i32_noret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB107_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_min_u32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB107_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw umin ptr %ptr, i32 %in seq_cst
  ret void
}

define void @flat_atomic_umin_i32_noret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_umin_i32_noret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB108_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_min_u32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB108_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i32_noret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB108_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_min_u32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB108_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i32_noret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB108_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_min_u32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB108_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw umin ptr %gep, i32 %in seq_cst
  ret void
}

define i32 @flat_atomic_umin_i32_ret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_umin_i32_ret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB109_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    v_min_u32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB109_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v0, v3
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i32_ret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB109_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    v_min_u32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB109_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v0, v3
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i32_ret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB109_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_min_u32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB109_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw umin ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define i32 @flat_atomic_umin_i32_ret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_umin_i32_ret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v0, v[3:4]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB110_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v1, v0
; GCN1-NEXT:    v_min_u32_e32 v0, v1, v2
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB110_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i32_ret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v0, v[3:4]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB110_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v1, v0
; GCN2-NEXT:    v_min_u32_e32 v0, v1, v2
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB110_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i32_ret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB110_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_min_u32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB110_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw umin ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx void @flat_atomic_umin_i32_noret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_umin_i32_noret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB111_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_min_u32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB111_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i32_noret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB111_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_min_u32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB111_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i32_noret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB111_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_min_u32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB111_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw umin ptr %ptr, i32 %in seq_cst
  ret void
}

define amdgpu_gfx void @flat_atomic_umin_i32_noret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_umin_i32_noret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB112_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_min_u32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB112_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i32_noret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB112_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_min_u32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB112_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i32_noret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB112_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_min_u32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB112_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw umin ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_gfx i32 @flat_atomic_umin_i32_ret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_umin_i32_ret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v0, v[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v1, s4
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:  .LBB113_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_min_u32_e32 v3, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB113_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i32_ret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v0, v[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v1, s4
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:  .LBB113_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_min_u32_e32 v3, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB113_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i32_ret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB113_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_min_u32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB113_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw umin ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx i32 @flat_atomic_umin_i32_ret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_umin_i32_ret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v1, s34
; GCN1-NEXT:    v_mov_b32_e32 v2, s35
; GCN1-NEXT:    flat_load_dword v0, v[1:2]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB114_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_min_u32_e32 v3, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB114_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i32_ret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v1, s34
; GCN2-NEXT:    v_mov_b32_e32 v2, s35
; GCN2-NEXT:    flat_load_dword v0, v[1:2]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB114_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_min_u32_e32 v3, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB114_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i32_ret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB114_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_min_u32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB114_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw umin ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define void @flat_umin_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_umin_i32_noret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_umin v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_umin_i32_noret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_umin v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_umin_i32_noret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_umin v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %tmp0 = atomicrmw umin ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define i32 @flat_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_umin v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_umin v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_umin v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %result = atomicrmw umin ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

; ---------------------------------------------------------------------
; atomicrmw min
; ---------------------------------------------------------------------

define void @flat_atomic_min_i32_noret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_min_i32_noret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB117_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_min_i32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB117_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i32_noret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB117_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_min_i32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB117_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i32_noret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB117_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_min_i32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB117_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw min ptr %ptr, i32 %in seq_cst
  ret void
}

define void @flat_atomic_min_i32_noret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_min_i32_noret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB118_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_min_i32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB118_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i32_noret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB118_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_min_i32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB118_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i32_noret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB118_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_min_i32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB118_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw min ptr %gep, i32 %in seq_cst
  ret void
}

define i32 @flat_atomic_min_i32_ret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_min_i32_ret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB119_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    v_min_i32_e32 v3, v4, v2
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB119_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v0, v3
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i32_ret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB119_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    v_min_i32_e32 v3, v4, v2
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB119_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v0, v3
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i32_ret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB119_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_min_i32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB119_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw min ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define i32 @flat_atomic_min_i32_ret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_min_i32_ret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v0, v[3:4]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB120_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v1, v0
; GCN1-NEXT:    v_min_i32_e32 v0, v1, v2
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB120_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i32_ret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v0, v[3:4]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB120_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v1, v0
; GCN2-NEXT:    v_min_i32_e32 v0, v1, v2
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB120_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i32_ret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB120_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_min_i32_e32 v3, v4, v2
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB120_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw min ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx void @flat_atomic_min_i32_noret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_min_i32_noret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB121_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_min_i32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB121_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i32_noret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB121_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_min_i32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB121_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i32_noret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB121_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_min_i32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB121_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw min ptr %ptr, i32 %in seq_cst
  ret void
}

define amdgpu_gfx void @flat_atomic_min_i32_noret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_min_i32_noret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB122_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_min_i32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB122_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i32_noret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB122_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_min_i32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB122_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i32_noret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB122_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_min_i32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB122_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw min ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_gfx i32 @flat_atomic_min_i32_ret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_min_i32_ret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v0, v[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v1, s4
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:  .LBB123_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_min_i32_e32 v3, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB123_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i32_ret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v0, v[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v1, s4
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:  .LBB123_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_min_i32_e32 v3, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB123_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i32_ret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB123_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_min_i32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB123_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw min ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx i32 @flat_atomic_min_i32_ret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_min_i32_ret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v1, s34
; GCN1-NEXT:    v_mov_b32_e32 v2, s35
; GCN1-NEXT:    flat_load_dword v0, v[1:2]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB124_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_min_i32_e32 v3, s6, v4
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB124_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i32_ret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v1, s34
; GCN2-NEXT:    v_mov_b32_e32 v2, s35
; GCN2-NEXT:    flat_load_dword v0, v[1:2]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB124_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_min_i32_e32 v3, s6, v4
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB124_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i32_ret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB124_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_min_i32_e32 v3, s6, v4
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB124_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw min ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr %out, i32 %in, i32 %index) {
; GCN1-LABEL: atomic_min_i32_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_ashr_i32 s5, s3, 31
; GCN1-NEXT:    s_mov_b32 s4, s3
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 16
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[0:1], 0
; GCN1-NEXT:  .LBB125_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_min_i32_e32 v2, s2, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT:    s_cbranch_execnz .LBB125_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i32_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_ashr_i32 s5, s3, 31
; GCN2-NEXT:    s_mov_b32 s4, s3
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 16
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[0:1], 0
; GCN2-NEXT:  .LBB125_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_min_i32_e32 v2, s2, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT:    s_cbranch_execnz .LBB125_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_endpgm
;
; GCN3-LABEL: atomic_min_i32_addr64_offset:
; GCN3:       ; %bb.0: ; %entry
; GCN3-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
; GCN3-NEXT:    s_ashr_i32 s5, s3, 31
; GCN3-NEXT:    s_mov_b32 s4, s3
; GCN3-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN3-NEXT:    s_add_u32 s0, s0, s4
; GCN3-NEXT:    s_addc_u32 s1, s1, s5
; GCN3-NEXT:    v_mov_b32_e32 v0, s0
; GCN3-NEXT:    v_mov_b32_e32 v1, s1
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[0:1], 0
; GCN3-NEXT:  .LBB125_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_min_i32_e32 v2, s2, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT:    s_cbranch_execnz .LBB125_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i32, ptr %out, i32 %index
  %gep = getelementptr inbounds i32, ptr %ptr, i32 4
  %tmp0 = atomicrmw min ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i32 %index) {
; GCN1-LABEL: atomic_min_i32_ret_addr64_offset:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_ashr_i32 s5, s7, 31
; GCN1-NEXT:    s_mov_b32 s4, s7
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    s_add_u32 s0, s0, 16
; GCN1-NEXT:    s_addc_u32 s1, s1, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dword v2, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[0:1], 0
; GCN1-NEXT:  .LBB126_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    v_min_i32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT:    s_cbranch_execnz .LBB126_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s3
; GCN1-NEXT:    flat_store_dword v[0:1], v2
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i32_ret_addr64_offset:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_ashr_i32 s5, s7, 31
; GCN2-NEXT:    s_mov_b32 s4, s7
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    s_add_u32 s0, s0, 16
; GCN2-NEXT:    s_addc_u32 s1, s1, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dword v2, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[0:1], 0
; GCN2-NEXT:  .LBB126_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    v_min_i32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT:    s_cbranch_execnz .LBB126_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s3
; GCN2-NEXT:    flat_store_dword v[0:1], v2
; GCN2-NEXT:    s_endpgm
;
; GCN3-LABEL: atomic_min_i32_ret_addr64_offset:
; GCN3:       ; %bb.0: ; %entry
; GCN3-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN3-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
; GCN3-NEXT:    s_ashr_i32 s5, s7, 31
; GCN3-NEXT:    s_mov_b32 s4, s7
; GCN3-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN3-NEXT:    s_add_u32 s0, s0, s4
; GCN3-NEXT:    s_addc_u32 s1, s1, s5
; GCN3-NEXT:    v_mov_b32_e32 v0, s0
; GCN3-NEXT:    v_mov_b32_e32 v1, s1
; GCN3-NEXT:    flat_load_dword v2, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[0:1], 0
; GCN3-NEXT:  .LBB126_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    v_min_i32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT:    s_cbranch_execnz .LBB126_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v0, s2
; GCN3-NEXT:    v_mov_b32_e32 v1, s3
; GCN3-NEXT:    flat_store_dword v[0:1], v2
; GCN3-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i32, ptr %out, i32 %index
  %gep = getelementptr inbounds i32, ptr %ptr, i32 4
  %tmp0 = atomicrmw min ptr %gep, i32 %in seq_cst
  store i32 %tmp0, ptr %out2
  ret void
}

define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_min_i32:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x9
; GCN1-NEXT:    s_load_dword s2, s[4:5], 0xb
; GCN1-NEXT:    s_mov_b64 s[0:1], 0
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s6
; GCN1-NEXT:    v_mov_b32_e32 v1, s7
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:  .LBB127_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_min_i32_e32 v2, s2, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT:    s_cbranch_execnz .LBB127_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i32:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
; GCN2-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GCN2-NEXT:    s_mov_b64 s[0:1], 0
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s6
; GCN2-NEXT:    v_mov_b32_e32 v1, s7
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:  .LBB127_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_min_i32_e32 v2, s2, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT:    s_cbranch_execnz .LBB127_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_endpgm
;
; GCN3-LABEL: atomic_min_i32:
; GCN3:       ; %bb.0: ; %entry
; GCN3-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
; GCN3-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GCN3-NEXT:    s_mov_b64 s[0:1], 0
; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s6
; GCN3-NEXT:    v_mov_b32_e32 v1, s7
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:  .LBB127_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_min_i32_e32 v2, s2, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT:    s_cbranch_execnz .LBB127_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_endpgm
entry:
  %tmp0 = atomicrmw min ptr %out, i32 %in seq_cst
  ret void
}

define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i32 %index) {
; GCN1-LABEL: atomic_min_i32_ret_addr64:
; GCN1:       ; %bb.0: ; %entry
; GCN1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
; GCN1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
; GCN1-NEXT:    s_ashr_i32 s5, s7, 31
; GCN1-NEXT:    s_mov_b32 s4, s7
; GCN1-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN1-NEXT:    s_add_u32 s0, s0, s4
; GCN1-NEXT:    s_addc_u32 s1, s1, s5
; GCN1-NEXT:    v_mov_b32_e32 v0, s0
; GCN1-NEXT:    v_mov_b32_e32 v1, s1
; GCN1-NEXT:    flat_load_dword v2, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[0:1], 0
; GCN1-NEXT:  .LBB128_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    v_min_i32_e32 v2, s6, v3
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT:    s_cbranch_execnz .LBB128_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v0, s2
; GCN1-NEXT:    v_mov_b32_e32 v1, s3
; GCN1-NEXT:    flat_store_dword v[0:1], v2
; GCN1-NEXT:    s_endpgm
;
; GCN2-LABEL: atomic_min_i32_ret_addr64:
; GCN2:       ; %bb.0: ; %entry
; GCN2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
; GCN2-NEXT:    s_ashr_i32 s5, s7, 31
; GCN2-NEXT:    s_mov_b32 s4, s7
; GCN2-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN2-NEXT:    s_add_u32 s0, s0, s4
; GCN2-NEXT:    s_addc_u32 s1, s1, s5
; GCN2-NEXT:    v_mov_b32_e32 v0, s0
; GCN2-NEXT:    v_mov_b32_e32 v1, s1
; GCN2-NEXT:    flat_load_dword v2, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[0:1], 0
; GCN2-NEXT:  .LBB128_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    v_min_i32_e32 v2, s6, v3
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT:    s_cbranch_execnz .LBB128_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v0, s2
; GCN2-NEXT:    v_mov_b32_e32 v1, s3
; GCN2-NEXT:    flat_store_dword v[0:1], v2
; GCN2-NEXT:    s_endpgm
;
; GCN3-LABEL: atomic_min_i32_ret_addr64:
; GCN3:       ; %bb.0: ; %entry
; GCN3-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN3-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT:    s_waitcnt lgkmcnt(0)
; GCN3-NEXT:    s_ashr_i32 s5, s7, 31
; GCN3-NEXT:    s_mov_b32 s4, s7
; GCN3-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
; GCN3-NEXT:    s_add_u32 s0, s0, s4
; GCN3-NEXT:    s_addc_u32 s1, s1, s5
; GCN3-NEXT:    v_mov_b32_e32 v0, s0
; GCN3-NEXT:    v_mov_b32_e32 v1, s1
; GCN3-NEXT:    flat_load_dword v2, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[0:1], 0
; GCN3-NEXT:  .LBB128_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    v_min_i32_e32 v2, s6, v3
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT:    s_cbranch_execnz .LBB128_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v0, s2
; GCN3-NEXT:    v_mov_b32_e32 v1, s3
; GCN3-NEXT:    flat_store_dword v[0:1], v2
; GCN3-NEXT:    s_endpgm
entry:
  %ptr = getelementptr inbounds i32, ptr %out, i32 %index
  %tmp0 = atomicrmw min ptr %ptr, i32 %in seq_cst
  store i32 %tmp0, ptr %out2
  ret void
}

define void @flat_min_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_min_i32_noret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_smin v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_min_i32_noret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_smin v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_min_i32_noret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_smin v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %tmp0 = atomicrmw min ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define i32 @flat_atomic_min_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_min_i32_ret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_smin v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i32_ret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_smin v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i32_ret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_smin v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %result = atomicrmw min ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

; ---------------------------------------------------------------------
; atomicrmw uinc_wrap
; ---------------------------------------------------------------------

define void @flat_atomic_uinc_wrap_i32_noret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i32_noret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB131_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 1, v4
; GCN1-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
; GCN1-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB131_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i32_noret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB131_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 1, v4
; GCN2-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
; GCN2-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB131_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i32_noret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB131_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_add_u32_e32 v3, 1, v4
; GCN3-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
; GCN3-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB131_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw uinc_wrap ptr %ptr, i32 %in seq_cst
  ret void
}

define void @flat_atomic_uinc_wrap_i32_noret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i32_noret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB132_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 1, v4
; GCN1-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
; GCN1-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB132_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i32_noret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB132_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 1, v4
; GCN2-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
; GCN2-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB132_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i32_noret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB132_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_add_u32_e32 v3, 1, v4
; GCN3-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
; GCN3-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB132_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw uinc_wrap ptr %gep, i32 %in seq_cst
  ret void
}

define i32 @flat_atomic_uinc_wrap_i32_ret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i32_ret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB133_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 1, v4
; GCN1-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
; GCN1-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB133_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    v_mov_b32_e32 v0, v3
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i32_ret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB133_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 1, v4
; GCN2-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
; GCN2-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB133_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    v_mov_b32_e32 v0, v3
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i32_ret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB133_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_add_u32_e32 v3, 1, v4
; GCN3-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
; GCN3-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB133_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw uinc_wrap ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define i32 @flat_atomic_uinc_wrap_i32_ret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i32_ret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v0, v[3:4]
; GCN1-NEXT:    s_mov_b64 s[4:5], 0
; GCN1-NEXT:  .LBB134_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v1, v0
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 1, v1
; GCN1-NEXT:    v_cmp_lt_u32_e32 vcc, v1, v2
; GCN1-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_cbranch_execnz .LBB134_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i32_ret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v0, v[3:4]
; GCN2-NEXT:    s_mov_b64 s[4:5], 0
; GCN2-NEXT:  .LBB134_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v1, v0
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 1, v1
; GCN2-NEXT:    v_cmp_lt_u32_e32 vcc, v1, v2
; GCN2-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_cbranch_execnz .LBB134_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i32_ret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[4:5], 0
; GCN3-NEXT:  .LBB134_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_add_u32_e32 v3, 1, v4
; GCN3-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
; GCN3-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT:    s_cbranch_execnz .LBB134_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw uinc_wrap ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx void @flat_atomic_uinc_wrap_i32_noret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i32_noret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB135_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v2, vcc, 1, v3
; GCN1-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
; GCN1-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB135_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i32_noret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB135_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v2, vcc, 1, v3
; GCN2-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
; GCN2-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB135_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i32_noret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB135_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_add_u32_e32 v2, 1, v3
; GCN3-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
; GCN3-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB135_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw uinc_wrap ptr %ptr, i32 %in seq_cst
  ret void
}

define amdgpu_gfx void @flat_atomic_uinc_wrap_i32_noret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i32_noret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB136_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v2, vcc, 1, v3
; GCN1-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
; GCN1-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB136_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i32_noret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB136_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v2, vcc, 1, v3
; GCN2-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
; GCN2-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB136_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i32_noret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:  .LBB136_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_add_u32_e32 v2, 1, v3
; GCN3-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
; GCN3-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    v_mov_b32_e32 v3, v2
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB136_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw uinc_wrap ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_gfx i32 @flat_atomic_uinc_wrap_i32_ret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i32_ret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v0, v[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v1, s4
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:  .LBB137_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 1, v4
; GCN1-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v4
; GCN1-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB137_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i32_ret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v0, v[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v1, s4
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:  .LBB137_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 1, v4
; GCN2-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v4
; GCN2-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB137_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i32_ret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB137_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_add_u32_e32 v0, 1, v4
; GCN3-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v4
; GCN3-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB137_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw uinc_wrap ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx i32 @flat_atomic_uinc_wrap_i32_ret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i32_ret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v1, s34
; GCN1-NEXT:    v_mov_b32_e32 v2, s35
; GCN1-NEXT:    flat_load_dword v0, v[1:2]
; GCN1-NEXT:    s_mov_b64 s[34:35], 0
; GCN1-NEXT:  .LBB138_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v0
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 1, v4
; GCN1-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v4
; GCN1-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_cbranch_execnz .LBB138_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i32_ret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v1, s34
; GCN2-NEXT:    v_mov_b32_e32 v2, s35
; GCN2-NEXT:    flat_load_dword v0, v[1:2]
; GCN2-NEXT:    s_mov_b64 s[34:35], 0
; GCN2-NEXT:  .LBB138_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v0
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 1, v4
; GCN2-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v4
; GCN2-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_cbranch_execnz .LBB138_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i32_ret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[34:35], 0
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB138_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v0
; GCN3-NEXT:    v_add_u32_e32 v0, 1, v4
; GCN3-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v4
; GCN3-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_cbranch_execnz .LBB138_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw uinc_wrap ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define void @flat_uinc_wrap_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_uinc_wrap_i32_noret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_inc v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_uinc_wrap_i32_noret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_inc v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_uinc_wrap_i32_noret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_inc v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %tmp0 = atomicrmw uinc_wrap ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define i32 @flat_atomic_uinc_wrap_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_uinc_wrap_i32_ret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i32_ret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i32_ret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_inc v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %result = atomicrmw uinc_wrap ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

; ---------------------------------------------------------------------
; atomicrmw udec_wrap
; ---------------------------------------------------------------------

define void @flat_atomic_udec_wrap_i32_noret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i32_noret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:  .LBB141_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_subrev_i32_e32 v3, vcc, 1, v4
; GCN1-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[4:5]
; GCN1-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_execnz .LBB141_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i32_noret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:  .LBB141_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_subrev_u32_e32 v3, vcc, 1, v4
; GCN2-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[4:5]
; GCN2-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_execnz .LBB141_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i32_noret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[6:7], 0
; GCN3-NEXT:  .LBB141_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_subrev_co_u32_e32 v3, vcc, 1, v4
; GCN3-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
; GCN3-NEXT:    s_or_b64 vcc, vcc, s[4:5]
; GCN3-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
; GCN3-NEXT:    s_cbranch_execnz .LBB141_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw udec_wrap ptr %ptr, i32 %in seq_cst
  ret void
}

define void @flat_atomic_udec_wrap_i32_noret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i32_noret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v4, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:  .LBB142_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_subrev_i32_e32 v3, vcc, 1, v4
; GCN1-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[4:5]
; GCN1-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_execnz .LBB142_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i32_noret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v4, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:  .LBB142_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_subrev_u32_e32 v3, vcc, 1, v4
; GCN2-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[4:5]
; GCN2-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_execnz .LBB142_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i32_noret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[6:7], 0
; GCN3-NEXT:  .LBB142_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_subrev_co_u32_e32 v3, vcc, 1, v4
; GCN3-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
; GCN3-NEXT:    s_or_b64 vcc, vcc, s[4:5]
; GCN3-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
; GCN3-NEXT:    s_cbranch_execnz .LBB142_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw udec_wrap ptr %gep, i32 %in seq_cst
  ret void
}

define i32 @flat_atomic_udec_wrap_i32_ret(ptr %ptr, i32 %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i32_ret:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:  .LBB143_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v4, v3
; GCN1-NEXT:    v_subrev_i32_e32 v3, vcc, 1, v4
; GCN1-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[4:5]
; GCN1-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_execnz .LBB143_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
; GCN1-NEXT:    v_mov_b32_e32 v0, v3
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i32_ret:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:  .LBB143_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v4, v3
; GCN2-NEXT:    v_subrev_u32_e32 v3, vcc, 1, v4
; GCN2-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[4:5]
; GCN2-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_execnz .LBB143_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
; GCN2-NEXT:    v_mov_b32_e32 v0, v3
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i32_ret:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1]
; GCN3-NEXT:    s_mov_b64 s[6:7], 0
; GCN3-NEXT:  .LBB143_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_subrev_co_u32_e32 v3, vcc, 1, v4
; GCN3-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
; GCN3-NEXT:    s_or_b64 vcc, vcc, s[4:5]
; GCN3-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
; GCN3-NEXT:    s_cbranch_execnz .LBB143_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw udec_wrap ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define i32 @flat_atomic_udec_wrap_i32_ret_offset(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i32_ret_offset:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_load_dword v0, v[3:4]
; GCN1-NEXT:    s_mov_b64 s[6:7], 0
; GCN1-NEXT:  .LBB144_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v1, v0
; GCN1-NEXT:    v_subrev_i32_e32 v0, vcc, 1, v1
; GCN1-NEXT:    v_cmp_gt_u32_e64 s[4:5], v1, v2
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[4:5]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
; GCN1-NEXT:    s_cbranch_execnz .LBB144_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i32_ret_offset:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_load_dword v0, v[3:4]
; GCN2-NEXT:    s_mov_b64 s[6:7], 0
; GCN2-NEXT:  .LBB144_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v1, v0
; GCN2-NEXT:    v_subrev_u32_e32 v0, vcc, 1, v1
; GCN2-NEXT:    v_cmp_gt_u32_e64 s[4:5], v1, v2
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[4:5]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
; GCN2-NEXT:    s_cbranch_execnz .LBB144_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i32_ret_offset:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT:    s_mov_b64 s[6:7], 0
; GCN3-NEXT:  .LBB144_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v4, v3
; GCN3-NEXT:    v_subrev_co_u32_e32 v3, vcc, 1, v4
; GCN3-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
; GCN3-NEXT:    s_or_b64 vcc, vcc, s[4:5]
; GCN3-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
; GCN3-NEXT:    s_cbranch_execnz .LBB144_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
; GCN3-NEXT:    v_mov_b32_e32 v0, v3
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw udec_wrap ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx void @flat_atomic_udec_wrap_i32_noret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i32_noret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v1, v[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v2, s4
; GCN1-NEXT:    s_mov_b64 s[36:37], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s6
; GCN1-NEXT:    v_mov_b32_e32 v3, s5
; GCN1-NEXT:  .LBB145_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_subrev_i32_e32 v0, vcc, 1, v1
; GCN1-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v1
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[34:35]
; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT:    v_mov_b32_e32 v1, v0
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT:    s_cbranch_execnz .LBB145_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i32_noret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v1, v[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v2, s4
; GCN2-NEXT:    s_mov_b64 s[36:37], 0
; GCN2-NEXT:    v_mov_b32_e32 v4, s6
; GCN2-NEXT:    v_mov_b32_e32 v3, s5
; GCN2-NEXT:  .LBB145_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_subrev_u32_e32 v0, vcc, 1, v1
; GCN2-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v1
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[34:35]
; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT:    v_mov_b32_e32 v1, v0
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT:    s_cbranch_execnz .LBB145_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i32_noret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v1, v[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v2, s4
; GCN3-NEXT:    s_mov_b64 s[36:37], 0
; GCN3-NEXT:    v_mov_b32_e32 v4, s6
; GCN3-NEXT:    v_mov_b32_e32 v3, s5
; GCN3-NEXT:  .LBB145_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_subrev_co_u32_e32 v0, vcc, 1, v1
; GCN3-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v1
; GCN3-NEXT:    s_or_b64 vcc, vcc, s[34:35]
; GCN3-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT:    v_mov_b32_e32 v1, v0
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT:    s_cbranch_execnz .LBB145_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %tmp0 = atomicrmw udec_wrap ptr %ptr, i32 %in seq_cst
  ret void
}

define amdgpu_gfx void @flat_atomic_udec_wrap_i32_noret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i32_noret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v0, s34
; GCN1-NEXT:    v_mov_b32_e32 v1, s35
; GCN1-NEXT:    flat_load_dword v3, v[0:1]
; GCN1-NEXT:    s_mov_b64 s[36:37], 0
; GCN1-NEXT:    v_mov_b32_e32 v4, s6
; GCN1-NEXT:  .LBB146_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_subrev_i32_e32 v2, vcc, 1, v3
; GCN1-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v3
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[34:35]
; GCN1-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT:    v_mov_b32_e32 v3, v2
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT:    s_cbranch_execnz .LBB146_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i32_noret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v0, s34
; GCN2-NEXT:    v_mov_b32_e32 v1, s35
; GCN2-NEXT:    flat_load_dword v3, v[0:1]
; GCN2-NEXT:    s_mov_b64 s[36:37], 0
; GCN2-NEXT:    v_mov_b32_e32 v4, s6
; GCN2-NEXT:  .LBB146_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_subrev_u32_e32 v2, vcc, 1, v3
; GCN2-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v3
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[34:35]
; GCN2-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT:    v_mov_b32_e32 v3, v2
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT:    s_cbranch_execnz .LBB146_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i32_noret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v1, v[0:1] offset:16
; GCN3-NEXT:    v_mov_b32_e32 v2, s4
; GCN3-NEXT:    s_mov_b64 s[36:37], 0
; GCN3-NEXT:    v_mov_b32_e32 v4, s6
; GCN3-NEXT:    v_mov_b32_e32 v3, s5
; GCN3-NEXT:  .LBB146_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_subrev_co_u32_e32 v0, vcc, 1, v1
; GCN3-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v1
; GCN3-NEXT:    s_or_b64 vcc, vcc, s[34:35]
; GCN3-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT:    v_mov_b32_e32 v1, v0
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT:    s_cbranch_execnz .LBB146_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %tmp0 = atomicrmw udec_wrap ptr %gep, i32 %in seq_cst
  ret void
}

define amdgpu_gfx i32 @flat_atomic_udec_wrap_i32_ret_scalar(ptr inreg %ptr, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i32_ret_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v0, s4
; GCN1-NEXT:    v_mov_b32_e32 v1, s5
; GCN1-NEXT:    flat_load_dword v0, v[0:1]
; GCN1-NEXT:    v_mov_b32_e32 v1, s4
; GCN1-NEXT:    s_mov_b64 s[36:37], 0
; GCN1-NEXT:    v_mov_b32_e32 v3, s6
; GCN1-NEXT:    v_mov_b32_e32 v2, s5
; GCN1-NEXT:  .LBB147_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v5, v0
; GCN1-NEXT:    v_subrev_i32_e32 v0, vcc, 1, v5
; GCN1-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v5
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[34:35]
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v5
; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT:    s_cbranch_execnz .LBB147_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i32_ret_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v0, s4
; GCN2-NEXT:    v_mov_b32_e32 v1, s5
; GCN2-NEXT:    flat_load_dword v0, v[0:1]
; GCN2-NEXT:    v_mov_b32_e32 v1, s4
; GCN2-NEXT:    s_mov_b64 s[36:37], 0
; GCN2-NEXT:    v_mov_b32_e32 v3, s6
; GCN2-NEXT:    v_mov_b32_e32 v2, s5
; GCN2-NEXT:  .LBB147_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v5, v0
; GCN2-NEXT:    v_subrev_u32_e32 v0, vcc, 1, v5
; GCN2-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v5
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[34:35]
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v5
; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT:    s_cbranch_execnz .LBB147_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i32_ret_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1]
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[36:37], 0
; GCN3-NEXT:    v_mov_b32_e32 v3, s6
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB147_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v5, v0
; GCN3-NEXT:    v_subrev_co_u32_e32 v0, vcc, 1, v5
; GCN3-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v5
; GCN3-NEXT:    s_or_b64 vcc, vcc, s[34:35]
; GCN3-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v5
; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT:    s_cbranch_execnz .LBB147_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %result = atomicrmw udec_wrap ptr %ptr, i32 %in seq_cst
  ret i32 %result
}

define amdgpu_gfx i32 @flat_atomic_udec_wrap_i32_ret_offset_scalar(ptr inreg %out, i32 inreg %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i32_ret_offset_scalar:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    s_add_u32 s34, s4, 16
; GCN1-NEXT:    s_addc_u32 s35, s5, 0
; GCN1-NEXT:    v_mov_b32_e32 v1, s34
; GCN1-NEXT:    v_mov_b32_e32 v2, s35
; GCN1-NEXT:    flat_load_dword v0, v[1:2]
; GCN1-NEXT:    s_mov_b64 s[36:37], 0
; GCN1-NEXT:    v_mov_b32_e32 v3, s6
; GCN1-NEXT:  .LBB148_1: ; %atomicrmw.start
; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_mov_b32_e32 v5, v0
; GCN1-NEXT:    v_subrev_i32_e32 v0, vcc, 1, v5
; GCN1-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v5
; GCN1-NEXT:    s_or_b64 vcc, vcc, s[34:35]
; GCN1-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc
; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v5
; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT:    s_cbranch_execnz .LBB148_1
; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i32_ret_offset_scalar:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    s_add_u32 s34, s4, 16
; GCN2-NEXT:    s_addc_u32 s35, s5, 0
; GCN2-NEXT:    v_mov_b32_e32 v1, s34
; GCN2-NEXT:    v_mov_b32_e32 v2, s35
; GCN2-NEXT:    flat_load_dword v0, v[1:2]
; GCN2-NEXT:    s_mov_b64 s[36:37], 0
; GCN2-NEXT:    v_mov_b32_e32 v3, s6
; GCN2-NEXT:  .LBB148_1: ; %atomicrmw.start
; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_mov_b32_e32 v5, v0
; GCN2-NEXT:    v_subrev_u32_e32 v0, vcc, 1, v5
; GCN2-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v5
; GCN2-NEXT:    s_or_b64 vcc, vcc, s[34:35]
; GCN2-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc
; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v5
; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT:    s_cbranch_execnz .LBB148_1
; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i32_ret_offset_scalar:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v0, s4
; GCN3-NEXT:    v_mov_b32_e32 v1, s5
; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT:    v_mov_b32_e32 v1, s4
; GCN3-NEXT:    s_mov_b64 s[36:37], 0
; GCN3-NEXT:    v_mov_b32_e32 v3, s6
; GCN3-NEXT:    v_mov_b32_e32 v2, s5
; GCN3-NEXT:  .LBB148_1: ; %atomicrmw.start
; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    v_mov_b32_e32 v5, v0
; GCN3-NEXT:    v_subrev_co_u32_e32 v0, vcc, 1, v5
; GCN3-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v5
; GCN3-NEXT:    s_or_b64 vcc, vcc, s[34:35]
; GCN3-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc
; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v5
; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT:    s_cbranch_execnz .LBB148_1
; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i32 4
  %result = atomicrmw udec_wrap ptr %gep, i32 %in seq_cst
  ret i32 %result
}

define void @flat_udec_wrap_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_udec_wrap_i32_noret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_dec v[0:1], v2
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_udec_wrap_i32_noret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_dec v[0:1], v2
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_udec_wrap_i32_noret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_dec v[0:1], v2 offset:16
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %tmp0 = atomicrmw udec_wrap ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret void
}

define i32 @flat_atomic_udec_wrap_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i32 %in) {
; GCN1-LABEL: flat_atomic_udec_wrap_i32_ret_offset__amdgpu_no_remote_memory:
; GCN1:       ; %bb.0:
; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN1-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT:    buffer_wbinvl1_vol
; GCN1-NEXT:    s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i32_ret_offset__amdgpu_no_remote_memory:
; GCN2:       ; %bb.0:
; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GCN2-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT:    buffer_wbinvl1_vol
; GCN2-NEXT:    s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i32_ret_offset__amdgpu_no_remote_memory:
; GCN3:       ; %bb.0:
; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT:    flat_atomic_dec v0, v[0:1], v2 offset:16 glc
; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT:    buffer_wbinvl1_vol
; GCN3-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr %out, i64 4
  %result = atomicrmw udec_wrap ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
  ret i32 %result
}

!0 = !{}