; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri < %s | FileCheck -check-prefixes=GCN,CI %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-TRUE16 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s

define i8 @atomic_load_monotonic_i8(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_i8:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_u8 v0, v0
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_i8:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_u8 v0, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: atomic_load_monotonic_i8:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT:    ds_load_u8_d16 v0, v0
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: atomic_load_monotonic_i8:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT:    ds_load_u8 v0, v0
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_i8:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_u8 v0, v0
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %load = load atomic i8, ptr addrspace(3) %ptr monotonic, align 1
  ret i8 %load
}

define i8 @atomic_load_monotonic_i8_offset(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_i8_offset:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_u8 v0, v0 offset:16
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_i8_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_u8 v0, v0 offset:16
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: atomic_load_monotonic_i8_offset:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT:    ds_load_u8_d16 v0, v0 offset:16
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: atomic_load_monotonic_i8_offset:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT:    ds_load_u8 v0, v0 offset:16
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_i8_offset:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_u8 v0, v0 offset:16
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %gep = getelementptr inbounds i8, ptr addrspace(3) %ptr, i8 16
  %load = load atomic i8, ptr addrspace(3) %gep monotonic, align 1
  ret i8 %load
}

define i16 @atomic_load_monotonic_i16(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_i16:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_u16 v0, v0
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_i16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_u16 v0, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: atomic_load_monotonic_i16:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT:    ds_load_u16_d16 v0, v0
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: atomic_load_monotonic_i16:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT:    ds_load_u16 v0, v0
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_i16:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_u16 v0, v0
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %load = load atomic i16, ptr addrspace(3) %ptr monotonic, align 2
  ret i16 %load
}

define i16 @atomic_load_monotonic_i16_offset(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_i16_offset:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_u16 v0, v0 offset:32
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_i16_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_u16 v0, v0 offset:32
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: atomic_load_monotonic_i16_offset:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT:    ds_load_u16_d16 v0, v0 offset:32
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: atomic_load_monotonic_i16_offset:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT:    ds_load_u16 v0, v0 offset:32
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_i16_offset:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_u16 v0, v0 offset:32
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %gep = getelementptr inbounds i16, ptr addrspace(3) %ptr, i16 16
  %load = load atomic i16, ptr addrspace(3) %gep monotonic, align 2
  ret i16 %load
}

define i32 @atomic_load_monotonic_i32(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_i32:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_b32 v0, v0
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_i32:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_b32 v0, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_monotonic_i32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    ds_load_b32 v0, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_i32:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_b32 v0, v0
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %load = load atomic i32, ptr addrspace(3) %ptr monotonic, align 4
  ret i32 %load
}

define i32 @atomic_load_monotonic_i32_offset(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_i32_offset:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_b32 v0, v0 offset:64
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_i32_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_b32 v0, v0 offset:64
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_monotonic_i32_offset:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    ds_load_b32 v0, v0 offset:64
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_i32_offset:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_b32 v0, v0 offset:64
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %gep = getelementptr inbounds i32, ptr addrspace(3) %ptr, i32 16
  %load = load atomic i32, ptr addrspace(3) %gep monotonic, align 4
  ret i32 %load
}

define i64 @atomic_load_monotonic_i64(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_i64:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_b64 v[0:1], v0
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_i64:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_b64 v[0:1], v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_monotonic_i64:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    ds_load_b64 v[0:1], v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_i64:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_b64 v[0:1], v0
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %load = load atomic i64, ptr addrspace(3) %ptr monotonic, align 8
  ret i64 %load
}

define i64 @atomic_load_monotonic_i64_offset(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_i64_offset:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_b64 v[0:1], v0 offset:128
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_i64_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 offset:128
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_monotonic_i64_offset:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 offset:128
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_i64_offset:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_b64 v[0:1], v0 offset:128
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %gep = getelementptr inbounds i64, ptr addrspace(3) %ptr, i32 16
  %load = load atomic i64, ptr addrspace(3) %gep monotonic, align 8
  ret i64 %load
}

define float @atomic_load_monotonic_f32_offset(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_f32_offset:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_b32 v0, v0 offset:64
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_f32_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_b32 v0, v0 offset:64
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_monotonic_f32_offset:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    ds_load_b32 v0, v0 offset:64
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_f32_offset:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_b32 v0, v0 offset:64
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %gep = getelementptr inbounds float, ptr addrspace(3) %ptr, i32 16
  %load = load atomic float, ptr addrspace(3) %gep monotonic, align 4
  ret float %load
}

define double @atomic_load_monotonic_f64_offset(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_f64_offset:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_b64 v[0:1], v0 offset:128
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_f64_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 offset:128
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_monotonic_f64_offset:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 offset:128
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_f64_offset:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_b64 v[0:1], v0 offset:128
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %gep = getelementptr inbounds double, ptr addrspace(3) %ptr, i32 16
  %load = load atomic double, ptr addrspace(3) %gep monotonic, align 8
  ret double %load
}

define ptr @atomic_load_monotonic_p0i8_offset(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_p0i8_offset:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_b64 v[0:1], v0 offset:128
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_p0i8_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 offset:128
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_monotonic_p0i8_offset:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 offset:128
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_p0i8_offset:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_b64 v[0:1], v0 offset:128
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %gep = getelementptr inbounds ptr, ptr addrspace(3) %ptr, i32 16
  %load = load atomic ptr, ptr addrspace(3) %gep monotonic, align 8
  ret ptr %load
}

define ptr addrspace(3) @atomic_load_monotonic_p3i8_offset(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_p3i8_offset:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_b32 v0, v0 offset:64
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_p3i8_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_b32 v0, v0 offset:64
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_monotonic_p3i8_offset:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    ds_load_b32 v0, v0 offset:64
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_p3i8_offset:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_b32 v0, v0 offset:64
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %gep = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) %ptr, i32 16
  %load = load atomic ptr addrspace(3), ptr addrspace(3) %gep monotonic, align 4
  ret ptr addrspace(3) %load
}

define i16 @atomic_load_monotonic_f16(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_f16:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_u16 v0, v0
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_u16 v0, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: atomic_load_monotonic_f16:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT:    ds_load_u16_d16 v0, v0
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: atomic_load_monotonic_f16:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT:    ds_load_u16 v0, v0
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_f16:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_u16 v0, v0
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %load = load atomic half, ptr addrspace(3) %ptr monotonic, align 2
  %ret = bitcast half %load to i16
  ret i16 %ret
}

define i16 @atomic_load_monotonic_f16_offset(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_f16_offset:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_u16 v0, v0 offset:32
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_f16_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_u16 v0, v0 offset:32
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: atomic_load_monotonic_f16_offset:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT:    ds_load_u16_d16 v0, v0 offset:32
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: atomic_load_monotonic_f16_offset:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT:    ds_load_u16 v0, v0 offset:32
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_f16_offset:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_u16 v0, v0 offset:32
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %gep = getelementptr inbounds half, ptr addrspace(3) %ptr, i32 16
  %load = load atomic half, ptr addrspace(3) %gep monotonic, align 2
  %ret = bitcast half %load to i16
  ret i16 %ret
}

define i16 @atomic_load_monotonic_bf16(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_bf16:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_u16 v0, v0
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_bf16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_u16 v0, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: atomic_load_monotonic_bf16:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT:    ds_load_u16_d16 v0, v0
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: atomic_load_monotonic_bf16:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT:    ds_load_u16 v0, v0
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_bf16:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_u16 v0, v0
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %load = load atomic bfloat, ptr addrspace(3) %ptr monotonic, align 2
  %ret = bitcast bfloat %load to i16
  ret i16 %ret
}

define i16 @atomic_load_monotonic_bf16_offset(ptr addrspace(3) %ptr) {
; CI-LABEL: atomic_load_monotonic_bf16_offset:
; CI:       ; %bb.0:
; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CI-NEXT:    s_mov_b32 m0, -1
; CI-NEXT:    ds_read_u16 v0, v0 offset:32
; CI-NEXT:    s_waitcnt lgkmcnt(0)
; CI-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_monotonic_bf16_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    ds_read_u16 v0, v0 offset:32
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: atomic_load_monotonic_bf16_offset:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT:    ds_load_u16_d16 v0, v0 offset:32
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: atomic_load_monotonic_bf16_offset:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT:    ds_load_u16 v0, v0 offset:32
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: atomic_load_monotonic_bf16_offset:
; GFX1250:       ; %bb.0:
; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT:    s_wait_kmcnt 0x0
; GFX1250-NEXT:    ds_load_u16 v0, v0 offset:32
; GFX1250-NEXT:    s_wait_dscnt 0x0
; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
  %gep = getelementptr inbounds bfloat, ptr addrspace(3) %ptr, i32 16
  %load = load atomic bfloat, ptr addrspace(3) %gep monotonic, align 2
  %ret = bitcast bfloat %load to i16
  ret i16 %ret
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
; GFX1250-FAKE16: {{.*}}
; GFX1250-TRUE16: {{.*}}