llvm/test/CodeGen/AMDGPU/gds-load-store.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu6.00 < %s | FileCheck -check-prefix=GFX6 %s
; RUN: llc -mtriple=amdgpu7.04 < %s | FileCheck -check-prefix=GFX7 %s
; RUN: llc -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -mtriple=amdgpu10.30 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s

define i32 @load_i32(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_i32:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b32 v0, v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b32 v0, v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_i32:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b32 v0, v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_i32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b32 v0, v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_i32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b32 v0, v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load i32, ptr addrspace(2) %ptr
  ret i32 %val
}

define i32 @load_i32_offset(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_i32_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 64, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b32 v0, v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_i32_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b32 v0, v0 offset:64 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_i32_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b32 v0, v0 offset:64 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_i32_offset:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b32 v0, v0 offset:64 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_i32_offset:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b32 v0, v0 offset:64 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr i32, ptr addrspace(2) %ptr, i32 16
  %val = load i32, ptr addrspace(2) %gep
  ret i32 %val
}

define i32 @sextload_i8(ptr addrspace(2) %ptr) {
; GFX6-LABEL: sextload_i8:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_i8 v0, v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: sextload_i8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_i8 v0, v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: sextload_i8:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_i8 v0, v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: sextload_i8:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_i8 v0, v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: sextload_i8:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_i8 v0, v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load i8, ptr addrspace(2) %ptr
  %ext = sext i8 %val to i32
  ret i32 %ext
}

define i32 @zextload_i8(ptr addrspace(2) %ptr) {
; GFX6-LABEL: zextload_i8:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_u8 v0, v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: zextload_i8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_u8 v0, v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: zextload_i8:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_u8 v0, v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: zextload_i8:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_u8 v0, v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: zextload_i8:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_u8 v0, v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load i8, ptr addrspace(2) %ptr
  %ext = zext i8 %val to i32
  ret i32 %ext
}

define i32 @sextload_i16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: sextload_i16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_i16 v0, v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: sextload_i16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_i16 v0, v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: sextload_i16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_i16 v0, v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: sextload_i16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_i16 v0, v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: sextload_i16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_i16 v0, v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load i16, ptr addrspace(2) %ptr
  %ext = sext i16 %val to i32
  ret i32 %ext
}

define i32 @zextload_i16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: zextload_i16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_u16 v0, v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: zextload_i16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_u16 v0, v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: zextload_i16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_u16 v0, v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: zextload_i16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_u16 v0, v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: zextload_i16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_u16 v0, v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load i16, ptr addrspace(2) %ptr
  %ext = zext i16 %val to i32
  ret i32 %ext
}

define i8 @load_i8(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_i8:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_u8 v0, v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_i8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_u8 v0, v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_i8:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_u8 v0, v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_i8:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_u8 v0, v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_i8:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_u8 v0, v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load i8, ptr addrspace(2) %ptr
  ret i8 %val
}

define half @load_f16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_u16 v0, v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_u16 v0, v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_u16 v0, v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_u16 v0, v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_u16 v0, v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load half, ptr addrspace(2) %ptr
  ret half %val
}

define i64 @load_i64_align4(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_i64_align4:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b32 v0, v0 gds
; GFX6-NEXT:    ds_read_b32 v1, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_i64_align4:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read2_b32 v[0:1], v0 offset1:1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_i64_align4:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read2_b32 v[0:1], v0 offset1:1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_i64_align4:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read2_b32 v[0:1], v0 offset1:1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_i64_align4:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_2addr_b32 v[0:1], v0 offset1:1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load i64, ptr addrspace(2) %ptr, align 4
  ret i64 %val
}

define i64 @load_i64_align8(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_i64_align8:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_i64_align8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_i64_align8:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_i64_align8:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_i64_align8:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load i64, ptr addrspace(2) %ptr, align 8
  ret i64 %val
}

define <3 x i32> @load_v3i32_align16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v3i32_align16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_mov_b32_e32 v2, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 8, v2
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    ds_read_b32 v2, v2 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v3i32_align16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b96 v[0:2], v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v3i32_align16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b96 v[0:2], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v3i32_align16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b96 v[0:2], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v3i32_align16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b96 v[0:2], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <3 x i32>, ptr addrspace(2) %ptr, align 16
  ret <3 x i32> %val
}

define <4 x i32> @load_v4i32_align8(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v4i32_align8:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_mov_b32_e32 v2, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 8, v2
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    ds_read_b64 v[2:3], v2 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v4i32_align8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v4i32_align8:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v4i32_align8:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v4i32_align8:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_2addr_b64 v[0:3], v0 offset1:1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <4 x i32>, ptr addrspace(2) %ptr, align 8
  ret <4 x i32> %val
}

define <4 x i32> @load_v4i32_align16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v4i32_align16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_mov_b32_e32 v2, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 8, v2
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    ds_read_b64 v[2:3], v2 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v4i32_align16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v4i32_align16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v4i32_align16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v4i32_align16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b128 v[0:3], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <4 x i32>, ptr addrspace(2) %ptr, align 16
  ret <4 x i32> %val
}

define void @store_i32(ptr addrspace(2) %ptr, i32 %val) {
; GFX6-LABEL: store_i32:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b32 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_i32:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b32 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_i32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b32 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_i32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b32 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store i32 %val, ptr addrspace(2) %ptr
  ret void
}

define void @store_i32_offset(ptr addrspace(2) %ptr, i32 %val) {
; GFX6-LABEL: store_i32_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 64, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_i32_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b32 v0, v1 offset:64 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_i32_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b32 v0, v1 offset:64 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_i32_offset:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b32 v0, v1 offset:64 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_i32_offset:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b32 v0, v1 offset:64 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr i32, ptr addrspace(2) %ptr, i32 16
  store i32 %val, ptr addrspace(2) %gep
  ret void
}

define void @truncstore_i32_to_i8(ptr addrspace(2) %ptr, i32 %val) {
; GFX6-LABEL: truncstore_i32_to_i8:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b8 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: truncstore_i32_to_i8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b8 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: truncstore_i32_to_i8:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b8 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: truncstore_i32_to_i8:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b8 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: truncstore_i32_to_i8:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b8 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %trunc = trunc i32 %val to i8
  store i8 %trunc, ptr addrspace(2) %ptr
  ret void
}

define void @truncstore_i32_to_i16(ptr addrspace(2) %ptr, i32 %val) {
; GFX6-LABEL: truncstore_i32_to_i16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b16 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: truncstore_i32_to_i16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b16 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: truncstore_i32_to_i16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b16 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: truncstore_i32_to_i16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b16 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: truncstore_i32_to_i16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b16 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %trunc = trunc i32 %val to i16
  store i16 %trunc, ptr addrspace(2) %ptr
  ret void
}

define void @store_i8(ptr addrspace(2) %ptr, i8 %val) {
; GFX6-LABEL: store_i8:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b8 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_i8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b8 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_i8:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b8 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_i8:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b8 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_i8:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b8 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store i8 %val, ptr addrspace(2) %ptr
  ret void
}

define void @store_f16(ptr addrspace(2) %ptr, half %val) {
; GFX6-LABEL: store_f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b16 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b16 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b16 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b16 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b16 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store half %val, ptr addrspace(2) %ptr
  ret void
}

define void @store_i64_align4(ptr addrspace(2) %ptr, i64 %val) {
; GFX6-LABEL: store_i64_align4:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v3, vcc, 4, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v3, v2 gds
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_i64_align4:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write2_b32 v0, v1, v2 offset1:1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_i64_align4:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write2_b32 v0, v1, v2 offset1:1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_i64_align4:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write2_b32 v0, v1, v2 offset1:1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_i64_align4:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_2addr_b32 v0, v1, v2 offset1:1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store i64 %val, ptr addrspace(2) %ptr, align 4
  ret void
}

define void @store_i64_align8(ptr addrspace(2) %ptr, i64 %val) {
; GFX6-LABEL: store_i64_align8:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_i64_align8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_i64_align8:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_i64_align8:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_i64_align8:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b64 v0, v[1:2] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store i64 %val, ptr addrspace(2) %ptr, align 8
  ret void
}

define void @store_v3i32_align16(ptr addrspace(2) %ptr, <3 x i32> %val) {
; GFX6-LABEL: store_v3i32_align16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v4, vcc, 8, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v4, v3 gds
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v3i32_align16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b96 v0, v[1:3] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v3i32_align16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b96 v0, v[1:3] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v3i32_align16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b96 v0, v[1:3] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v3i32_align16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b96 v0, v[1:3] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <3 x i32> %val, ptr addrspace(2) %ptr, align 16
  ret void
}

define void @store_v4i32_align8(ptr addrspace(2) %ptr, <4 x i32> %val) {
; GFX6-LABEL: store_v4i32_align8:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v5, vcc, 8, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v5, v[3:4] gds
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v4i32_align8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v4i32_align8:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v4i32_align8:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v4i32_align8:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_2addr_b64 v0, v[1:2], v[3:4] offset1:1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <4 x i32> %val, ptr addrspace(2) %ptr, align 8
  ret void
}

define void @store_v4i32_align16(ptr addrspace(2) %ptr, <4 x i32> %val) {
; GFX6-LABEL: store_v4i32_align16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v5, vcc, 8, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v5, v[3:4] gds
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v4i32_align16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v4i32_align16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v4i32_align16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v4i32_align16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b128 v0, v[1:4] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <4 x i32> %val, ptr addrspace(2) %ptr, align 16
  ret void
}

define i32 @atomic_load_i32(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_i32:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b32 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b32 v0, v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_i32:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b32 v0, v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_i32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_b32 v0, v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_i32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_b32 v0, v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load atomic i32, ptr addrspace(2) %ptr seq_cst, align 4
  ret i32 %val
}

define i16 @atomic_load_i16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_i16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_u16 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_i16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_u16 v0, v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_i16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_u16 v0, v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_i16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_u16 v0, v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_i16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_u16 v0, v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load atomic i16, ptr addrspace(2) %ptr seq_cst, align 2
  ret i16 %val
}

define i64 @atomic_load_i64(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_i64:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_i64:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_i64:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_i64:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_i64:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load atomic i64, ptr addrspace(2) %ptr seq_cst, align 8
  ret i64 %val
}

define void @atomic_store_i32(ptr addrspace(2) %ptr, i32 %val) {
; GFX6-LABEL: atomic_store_i32:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_i32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b32 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_i32:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b32 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_i32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b32 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_i32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b32 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store atomic i32 %val, ptr addrspace(2) %ptr seq_cst, align 4
  ret void
}

define void @atomic_store_i16(ptr addrspace(2) %ptr, i16 %val) {
; GFX6-LABEL: atomic_store_i16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b16 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_i16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b16 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_i16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b16 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_i16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b16 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_i16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b16 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store atomic i16 %val, ptr addrspace(2) %ptr seq_cst, align 2
  ret void
}

define void @atomic_store_i64(ptr addrspace(2) %ptr, i64 %val) {
; GFX6-LABEL: atomic_store_i64:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_i64:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_i64:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_i64:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_i64:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b64 v0, v[1:2] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store atomic i64 %val, ptr addrspace(2) %ptr seq_cst, align 8
  ret void
}
define i8 @atomic_load_i8(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_i8:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_u8 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_i8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_u8 v0, v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_i8:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_u8 v0, v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_i8:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_u8 v0, v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_i8:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_u8 v0, v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load atomic i8, ptr addrspace(2) %ptr seq_cst, align 1
  ret i8 %val
}

define i32 @atomic_load_zext_i8(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_zext_i8:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_u8 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_zext_i8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_u8 v0, v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_zext_i8:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_u8 v0, v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_zext_i8:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_u8 v0, v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: atomic_load_zext_i8:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_mov_b32 m0, 0
; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT:    ds_load_u8 v0, v0 gds
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    buffer_gl1_inv
; GFX11-TRUE16-NEXT:    buffer_gl0_inv
; GFX11-TRUE16-NEXT:    s_waitcnt expcnt(0)
; GFX11-TRUE16-NEXT:    v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: atomic_load_zext_i8:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT:    s_mov_b32 m0, 0
; GFX11-FAKE16-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-FAKE16-NEXT:    ds_load_u8 v0, v0 gds
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    buffer_gl1_inv
; GFX11-FAKE16-NEXT:    buffer_gl0_inv
; GFX11-FAKE16-NEXT:    s_waitcnt expcnt(0)
; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
  %load = load atomic i8, ptr addrspace(2) %ptr seq_cst, align 1
  %ext = zext i8 %load to i32
  ret i32 %ext
}

define i32 @atomic_load_sext_i8(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_sext_i8:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_u8 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 8
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_sext_i8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_u8 v0, v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    v_bfe_i32 v0, v0, 0, 8
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_sext_i8:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_u8 v0, v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 8
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_sext_i8:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_u8 v0, v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    v_bfe_i32 v0, v0, 0, 8
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_sext_i8:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_u8 v0, v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    v_bfe_i32 v0, v0, 0, 8
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %load = load atomic i8, ptr addrspace(2) %ptr seq_cst, align 1
  %ext = sext i8 %load to i32
  ret i32 %ext
}

define i32 @atomic_load_zext_i16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_zext_i16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_u16 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_zext_i16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_u16 v0, v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_zext_i16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_u16 v0, v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_zext_i16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_u16 v0, v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: atomic_load_zext_i16:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_mov_b32 m0, 0
; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT:    ds_load_u16 v0, v0 gds
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    buffer_gl1_inv
; GFX11-TRUE16-NEXT:    buffer_gl0_inv
; GFX11-TRUE16-NEXT:    s_waitcnt expcnt(0)
; GFX11-TRUE16-NEXT:    v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: atomic_load_zext_i16:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT:    s_mov_b32 m0, 0
; GFX11-FAKE16-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-FAKE16-NEXT:    ds_load_u16 v0, v0 gds
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    buffer_gl1_inv
; GFX11-FAKE16-NEXT:    buffer_gl0_inv
; GFX11-FAKE16-NEXT:    s_waitcnt expcnt(0)
; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
  %load = load atomic i16, ptr addrspace(2) %ptr seq_cst, align 2
  %ext = zext i16 %load to i32
  ret i32 %ext
}

define i32 @atomic_load_sext_i16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_sext_i16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_u16 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 16
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_sext_i16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_u16 v0, v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    v_bfe_i32 v0, v0, 0, 16
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_sext_i16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_u16 v0, v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 16
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_sext_i16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_u16 v0, v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    v_bfe_i32 v0, v0, 0, 16
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_sext_i16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_u16 v0, v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    v_bfe_i32 v0, v0, 0, 16
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %load = load atomic i16, ptr addrspace(2) %ptr seq_cst, align 2
  %ext = sext i16 %load to i32
  ret i32 %ext
}

define i32 @atomic_load_i32_offset(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_i32_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 64, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b32 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_i32_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b32 v0, v0 offset:64 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_i32_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b32 v0, v0 offset:64 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_i32_offset:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_b32 v0, v0 offset:64 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_i32_offset:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_b32 v0, v0 offset:64 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr addrspace(2) %ptr, i32 16
  %val = load atomic i32, ptr addrspace(2) %gep seq_cst, align 4
  ret i32 %val
}

define float @atomic_load_f32(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_f32:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b32 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_f32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b32 v0, v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_f32:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b32 v0, v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_f32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_b32 v0, v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_f32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_b32 v0, v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load atomic float, ptr addrspace(2) %ptr seq_cst, align 4
  ret float %val
}

define double @atomic_load_f64(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_f64:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_f64:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_f64:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_f64:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_f64:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load atomic double, ptr addrspace(2) %ptr seq_cst, align 8
  ret double %val
}

define <2 x i16> @atomic_load_v2i16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_v2i16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b32 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_v2i16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b32 v0, v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_v2i16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b32 v0, v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_v2i16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_b32 v0, v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_v2i16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_b32 v0, v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load atomic <2 x i16>, ptr addrspace(2) %ptr seq_cst, align 4
  ret <2 x i16> %val
}

define ptr addrspace(1) @atomic_load_p1(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_p1:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_p1:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_p1:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_p1:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_p1:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load atomic ptr addrspace(1), ptr addrspace(2) %ptr seq_cst, align 8
  ret ptr addrspace(1) %val
}

define ptr addrspace(2) @atomic_load_p2(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_p2:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b32 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_p2:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b32 v0, v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_p2:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b32 v0, v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_p2:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_b32 v0, v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_p2:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_b32 v0, v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load atomic ptr addrspace(2), ptr addrspace(2) %ptr seq_cst, align 4
  ret ptr addrspace(2) %val
}

define ptr addrspace(3) @atomic_load_p3(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_p3:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b32 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_p3:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b32 v0, v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_p3:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b32 v0, v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_p3:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_b32 v0, v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_p3:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_b32 v0, v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load atomic ptr addrspace(3), ptr addrspace(2) %ptr seq_cst, align 4
  ret ptr addrspace(3) %val
}

define void @atomic_store_i8(ptr addrspace(2) %ptr, i8 %val) {
; GFX6-LABEL: atomic_store_i8:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b8 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_i8:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b8 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_i8:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b8 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_i8:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b8 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_i8:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b8 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store atomic i8 %val, ptr addrspace(2) %ptr seq_cst, align 1
  ret void
}

define void @atomic_store_i32_offset(ptr addrspace(2) %ptr, i32 %val) {
; GFX6-LABEL: atomic_store_i32_offset:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 64, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_i32_offset:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b32 v0, v1 offset:64 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_i32_offset:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b32 v0, v1 offset:64 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_i32_offset:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b32 v0, v1 offset:64 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_i32_offset:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b32 v0, v1 offset:64 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %gep = getelementptr inbounds i32, ptr addrspace(2) %ptr, i32 16
  store atomic i32 %val, ptr addrspace(2) %gep seq_cst, align 4
  ret void
}

define void @atomic_store_f32(ptr addrspace(2) %ptr, float %val) {
; GFX6-LABEL: atomic_store_f32:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_f32:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b32 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_f32:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b32 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_f32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b32 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_f32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b32 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store atomic float %val, ptr addrspace(2) %ptr seq_cst, align 4
  ret void
}

define void @atomic_store_f64(ptr addrspace(2) %ptr, double %val) {
; GFX6-LABEL: atomic_store_f64:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_f64:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_f64:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_f64:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_f64:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b64 v0, v[1:2] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store atomic double %val, ptr addrspace(2) %ptr seq_cst, align 8
  ret void
}

define void @atomic_store_v2i16(ptr addrspace(2) %ptr, <2 x i16> %val) {
; GFX6-LABEL: atomic_store_v2i16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_v2i16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b32 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_v2i16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b32 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_v2i16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b32 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_v2i16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b32 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store atomic <2 x i16> %val, ptr addrspace(2) %ptr seq_cst, align 4
  ret void
}

define void @atomic_store_p1(ptr addrspace(2) %ptr, ptr addrspace(1) %val) {
; GFX6-LABEL: atomic_store_p1:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_p1:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_p1:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_p1:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_p1:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b64 v0, v[1:2] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store atomic ptr addrspace(1) %val, ptr addrspace(2) %ptr seq_cst, align 8
  ret void
}

define void @atomic_store_p2(ptr addrspace(2) %ptr, ptr addrspace(2) %val) {
; GFX6-LABEL: atomic_store_p2:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_p2:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b32 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_p2:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b32 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_p2:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b32 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_p2:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b32 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store atomic ptr addrspace(2) %val, ptr addrspace(2) %ptr seq_cst, align 4
  ret void
}

define void @atomic_store_p3(ptr addrspace(2) %ptr, ptr addrspace(3) %val) {
; GFX6-LABEL: atomic_store_p3:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_p3:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b32 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_p3:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b32 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_p3:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b32 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_p3:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b32 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store atomic ptr addrspace(3) %val, ptr addrspace(2) %ptr seq_cst, align 4
  ret void
}

define ptr addrspace(1) @load_p1(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_p1:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_p1:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_p1:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_p1:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_p1:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load ptr addrspace(1), ptr addrspace(2) %ptr, align 8
  ret ptr addrspace(1) %val
}

define ptr addrspace(2) @load_p2(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_p2:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b32 v0, v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_p2:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b32 v0, v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_p2:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b32 v0, v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_p2:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b32 v0, v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_p2:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b32 v0, v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load ptr addrspace(2), ptr addrspace(2) %ptr, align 4
  ret ptr addrspace(2) %val
}

define ptr addrspace(3) @load_p3(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_p3:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b32 v0, v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_p3:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b32 v0, v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_p3:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b32 v0, v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_p3:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b32 v0, v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_p3:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b32 v0, v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load ptr addrspace(3), ptr addrspace(2) %ptr, align 4
  ret ptr addrspace(3) %val
}

define <2 x ptr addrspace(1)> @load_v2p1(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v2p1:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_mov_b32_e32 v2, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 8, v2
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    ds_read_b64 v[2:3], v2 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v2p1:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v2p1:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v2p1:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v2p1:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b128 v[0:3], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <2 x ptr addrspace(1)>, ptr addrspace(2) %ptr, align 16
  ret <2 x ptr addrspace(1)> %val
}

define <2 x ptr addrspace(2)> @load_v2p2(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v2p2:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v2p2:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v2p2:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v2p2:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v2p2:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <2 x ptr addrspace(2)>, ptr addrspace(2) %ptr, align 8
  ret <2 x ptr addrspace(2)> %val
}

define <2 x ptr addrspace(3)> @load_v2p3(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v2p3:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v2p3:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v2p3:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v2p3:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v2p3:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <2 x ptr addrspace(3)>, ptr addrspace(2) %ptr, align 8
  ret <2 x ptr addrspace(3)> %val
}

define <3 x ptr addrspace(3)> @load_v3p3(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v3p3:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_mov_b32_e32 v2, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 8, v2
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    ds_read_b32 v2, v2 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v3p3:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b96 v[0:2], v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v3p3:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b96 v[0:2], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v3p3:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b96 v[0:2], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v3p3:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b96 v[0:2], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <3 x ptr addrspace(3)>, ptr addrspace(2) %ptr, align 16
  ret <3 x ptr addrspace(3)> %val
}

define <4 x ptr addrspace(3)> @load_v4p3(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v4p3:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_mov_b32_e32 v2, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 8, v2
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    ds_read_b64 v[2:3], v2 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v4p3:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v4p3:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v4p3:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v4p3:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b128 v[0:3], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <4 x ptr addrspace(3)>, ptr addrspace(2) %ptr, align 16
  ret <4 x ptr addrspace(3)> %val
}

define void @store_p1(ptr addrspace(2) %ptr, ptr addrspace(1) %val) {
; GFX6-LABEL: store_p1:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_p1:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_p1:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_p1:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_p1:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b64 v0, v[1:2] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store ptr addrspace(1) %val, ptr addrspace(2) %ptr, align 8
  ret void
}

define void @store_p2(ptr addrspace(2) %ptr, ptr addrspace(2) %val) {
; GFX6-LABEL: store_p2:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_p2:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b32 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_p2:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b32 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_p2:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b32 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_p2:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b32 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store ptr addrspace(2) %val, ptr addrspace(2) %ptr, align 4
  ret void
}

define void @store_p3(ptr addrspace(2) %ptr, ptr addrspace(3) %val) {
; GFX6-LABEL: store_p3:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_p3:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b32 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_p3:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b32 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_p3:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b32 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_p3:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b32 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store ptr addrspace(3) %val, ptr addrspace(2) %ptr, align 4
  ret void
}

define void @store_v2p1(ptr addrspace(2) %ptr, <2 x ptr addrspace(1)> %val) {
; GFX6-LABEL: store_v2p1:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v5, vcc, 8, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v5, v[3:4] gds
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v2p1:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v2p1:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v2p1:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v2p1:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b128 v0, v[1:4] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <2 x ptr addrspace(1)> %val, ptr addrspace(2) %ptr, align 16
  ret void
}

define void @store_v2p2(ptr addrspace(2) %ptr, <2 x ptr addrspace(2)> %val) {
; GFX6-LABEL: store_v2p2:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v2p2:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v2p2:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v2p2:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v2p2:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b64 v0, v[1:2] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <2 x ptr addrspace(2)> %val, ptr addrspace(2) %ptr, align 8
  ret void
}

define void @store_v2p3(ptr addrspace(2) %ptr, <2 x ptr addrspace(3)> %val) {
; GFX6-LABEL: store_v2p3:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v2p3:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v2p3:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v2p3:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v2p3:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b64 v0, v[1:2] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <2 x ptr addrspace(3)> %val, ptr addrspace(2) %ptr, align 8
  ret void
}

define void @store_v3p3(ptr addrspace(2) %ptr, <3 x ptr addrspace(3)> %val) {
; GFX6-LABEL: store_v3p3:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v4, vcc, 8, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v4, v3 gds
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v3p3:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b96 v0, v[1:3] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v3p3:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b96 v0, v[1:3] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v3p3:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b96 v0, v[1:3] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v3p3:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b96 v0, v[1:3] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <3 x ptr addrspace(3)> %val, ptr addrspace(2) %ptr, align 16
  ret void
}

define void @store_v4p3(ptr addrspace(2) %ptr, <4 x ptr addrspace(3)> %val) {
; GFX6-LABEL: store_v4p3:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v5, vcc, 8, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v5, v[3:4] gds
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v4p3:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v4p3:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v4p3:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v4p3:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b128 v0, v[1:4] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <4 x ptr addrspace(3)> %val, ptr addrspace(2) %ptr, align 16
  ret void
}

define bfloat @load_bf16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_bf16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_u16 v0, v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_bf16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_u16 v0, v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_bf16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_u16 v0, v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_bf16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_u16 v0, v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_bf16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_u16 v0, v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load bfloat, ptr addrspace(2) %ptr
  ret bfloat %val
}

define <2 x half> @load_v2f16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v2f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b32 v0, v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v2f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b32 v0, v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v2f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b32 v0, v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v2f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b32 v0, v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v2f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b32 v0, v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <2 x half>, ptr addrspace(2) %ptr
  ret <2 x half> %val
}

define <3 x half> @load_v3f16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v3f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v3f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v3f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v3f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v3f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <3 x half>, ptr addrspace(2) %ptr
  ret <3 x half> %val
}

define <4 x half> @load_v4f16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v4f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v4f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v4f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v4f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v4f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <4 x half>, ptr addrspace(2) %ptr
  ret <4 x half> %val
}

define <8 x half> @load_v8f16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v8f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_mov_b32_e32 v2, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 8, v2
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    ds_read_b64 v[2:3], v2 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v8f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v8f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v8f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v8f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b128 v[0:3], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <8 x half>, ptr addrspace(2) %ptr
  ret <8 x half> %val
}

define <2 x bfloat> @load_v2bf16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v2bf16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b32 v0, v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v2bf16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b32 v0, v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v2bf16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b32 v0, v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v2bf16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b32 v0, v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v2bf16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b32 v0, v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <2 x bfloat>, ptr addrspace(2) %ptr
  ret <2 x bfloat> %val
}

define <3 x bfloat> @load_v3bf16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v3bf16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v3bf16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v3bf16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v3bf16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v3bf16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <3 x bfloat>, ptr addrspace(2) %ptr
  ret <3 x bfloat> %val
}

define <4 x bfloat> @load_v4bf16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v4bf16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v4bf16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v4bf16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v4bf16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v4bf16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <4 x bfloat>, ptr addrspace(2) %ptr
  ret <4 x bfloat> %val
}

define <8 x bfloat> @load_v8bf16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: load_v8bf16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_mov_b32_e32 v2, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    v_add_i32_e32 v2, vcc, 8, v2
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    ds_read_b64 v[2:3], v2 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: load_v8bf16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: load_v8bf16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_v8bf16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_read_b128 v[0:3], v0 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: load_v8bf16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_load_b128 v[0:3], v0 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load <8 x bfloat>, ptr addrspace(2) %ptr
  ret <8 x bfloat> %val
}

define void @store_bf16(ptr addrspace(2) %ptr, bfloat %val) {
; GFX6-LABEL: store_bf16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b16 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_bf16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b16 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_bf16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b16 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_bf16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b16 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_bf16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b16 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store bfloat %val, ptr addrspace(2) %ptr
  ret void
}

define void @store_v2f16(ptr addrspace(2) %ptr, <2 x half> %val) {
; GFX6-LABEL: store_v2f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v2f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b32 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v2f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b32 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v2f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b32 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v2f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b32 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <2 x half> %val, ptr addrspace(2) %ptr
  ret void
}

define void @store_v3f16(ptr addrspace(2) %ptr, <3 x half> %val) {
; GFX6-LABEL: store_v3f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 4, v0
; GFX6-NEXT:    ds_write_b16 v0, v2 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v3f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b16 v0, v2 offset:4 gds
; GFX7-NEXT:    ds_write_b32 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v3f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b16 v0, v2 offset:4 gds
; GFX9-NEXT:    ds_write_b32 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v3f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b16 v0, v2 offset:4 gds
; GFX10-NEXT:    ds_write_b32 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v3f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b16 v0, v2 offset:4 gds
; GFX11-NEXT:    ds_store_b32 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <3 x half> %val, ptr addrspace(2) %ptr
  ret void
}

define void @store_v4f16(ptr addrspace(2) %ptr, <4 x half> %val) {
; GFX6-LABEL: store_v4f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v4f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v4f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v4f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v4f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b64 v0, v[1:2] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <4 x half> %val, ptr addrspace(2) %ptr
  ret void
}

define void @store_v8f16(ptr addrspace(2) %ptr, <8 x half> %val) {
; GFX6-LABEL: store_v8f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v5, vcc, 8, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v5, v[3:4] gds
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v8f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v8f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v8f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v8f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b128 v0, v[1:4] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <8 x half> %val, ptr addrspace(2) %ptr
  ret void
}

define void @store_v2bf16(ptr addrspace(2) %ptr, <2 x bfloat> %val) {
; GFX6-LABEL: store_v2bf16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v2bf16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b32 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v2bf16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b32 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v2bf16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b32 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v2bf16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b32 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <2 x bfloat> %val, ptr addrspace(2) %ptr
  ret void
}

define void @store_v3bf16(ptr addrspace(2) %ptr, <3 x bfloat> %val) {
; GFX6-LABEL: store_v3bf16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 4, v0
; GFX6-NEXT:    ds_write_b16 v0, v2 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v3bf16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b16 v0, v2 offset:4 gds
; GFX7-NEXT:    ds_write_b32 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v3bf16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b16 v0, v2 offset:4 gds
; GFX9-NEXT:    ds_write_b32 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v3bf16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b16 v0, v2 offset:4 gds
; GFX10-NEXT:    ds_write_b32 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v3bf16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b16 v0, v2 offset:4 gds
; GFX11-NEXT:    ds_store_b32 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <3 x bfloat> %val, ptr addrspace(2) %ptr
  ret void
}

define void @store_v4bf16(ptr addrspace(2) %ptr, <4 x bfloat> %val) {
; GFX6-LABEL: store_v4bf16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v4bf16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v4bf16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v4bf16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v4bf16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b64 v0, v[1:2] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <4 x bfloat> %val, ptr addrspace(2) %ptr
  ret void
}

define void @store_v8bf16(ptr addrspace(2) %ptr, <8 x bfloat> %val) {
; GFX6-LABEL: store_v8bf16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    v_add_i32_e32 v5, vcc, 8, v0
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v5, v[3:4] gds
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: store_v8bf16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: store_v8bf16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: store_v8bf16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    ds_write_b128 v0, v[1:4] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: store_v8bf16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    ds_store_b128 v0, v[1:4] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store <8 x bfloat> %val, ptr addrspace(2) %ptr
  ret void
}

define half @atomic_load_f16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_u16 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_u16 v0, v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_u16 v0, v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_u16 v0, v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_u16 v0, v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load atomic half, ptr addrspace(2) %ptr seq_cst, align 2
  ret half %val
}

define bfloat @atomic_load_bf16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_bf16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_u16 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_bf16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_u16 v0, v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_bf16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_u16 v0, v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_bf16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_u16 v0, v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_bf16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_u16 v0, v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load atomic bfloat, ptr addrspace(2) %ptr seq_cst, align 2
  ret bfloat %val
}

define <2 x half> @atomic_load_v2f16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_v2f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b32 v0, v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_v2f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b32 v0, v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_v2f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b32 v0, v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_v2f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_b32 v0, v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_v2f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_b32 v0, v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load atomic <2 x half>, ptr addrspace(2) %ptr seq_cst, align 4
  ret <2 x half> %val
}

define <4 x half> @atomic_load_v4f16(ptr addrspace(2) %ptr) {
; GFX6-LABEL: atomic_load_v4f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
; GFX6-NEXT:    buffer_wbinvl1
; GFX6-NEXT:    s_waitcnt expcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_v4f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
; GFX7-NEXT:    buffer_wbinvl1_vol
; GFX7-NEXT:    s_waitcnt expcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_v4f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    buffer_wbinvl1_vol
; GFX9-NEXT:    s_waitcnt expcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_load_v4f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_read_b64 v[0:1], v0 gds
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    s_waitcnt expcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_load_v4f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_load_b64 v[0:1], v0 gds
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    s_waitcnt expcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  %val = load atomic <4 x half>, ptr addrspace(2) %ptr seq_cst, align 8
  ret <4 x half> %val
}

define void @atomic_store_f16(ptr addrspace(2) %ptr, half %val) {
; GFX6-LABEL: atomic_store_f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b16 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b16 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b16 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b16 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b16 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store atomic half %val, ptr addrspace(2) %ptr seq_cst, align 2
  ret void
}

define void @atomic_store_bf16(ptr addrspace(2) %ptr, bfloat %val) {
; GFX6-LABEL: atomic_store_bf16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b16 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_bf16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b16 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_bf16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b16 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_bf16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b16 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_bf16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b16 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store atomic bfloat %val, ptr addrspace(2) %ptr seq_cst, align 2
  ret void
}

define void @atomic_store_v2f16(ptr addrspace(2) %ptr, <2 x half> %val) {
; GFX6-LABEL: atomic_store_v2f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b32 v0, v1 gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_v2f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b32 v0, v1 gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_v2f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b32 v0, v1 gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_v2f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b32 v0, v1 gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_v2f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b32 v0, v1 gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store atomic <2 x half> %val, ptr addrspace(2) %ptr seq_cst, align 4
  ret void
}

define void @atomic_store_v4f16(ptr addrspace(2) %ptr, <4 x half> %val) {
; GFX6-LABEL: atomic_store_v4f16:
; GFX6:       ; %bb.0:
; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_mov_b32 m0, 0
; GFX6-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX6-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX6-NEXT:    s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_store_v4f16:
; GFX7:       ; %bb.0:
; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_mov_b32 m0, 0
; GFX7-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX7-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX7-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_store_v4f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_mov_b32 m0, 0
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX9-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX9-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-LABEL: atomic_store_v4f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_mov_b32 m0, 0
; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX10-NEXT:    ds_write_b64 v0, v[1:2] gds
; GFX10-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX10-NEXT:    s_setpc_b64 s[30:31]
;
; GFX11-LABEL: atomic_store_v4f16:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_mov_b32 m0, 0
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    ds_store_b64 v0, v[1:2] gds
; GFX11-NEXT:    s_waitcnt expcnt(0) lgkmcnt(0)
; GFX11-NEXT:    s_setpc_b64 s[30:31]
  store atomic <4 x half> %val, ptr addrspace(2) %ptr seq_cst, align 8
  ret void
}