llvm/test/CodeGen/AMDGPU/addrspacecast-nonnull.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck --check-prefixes=CHECK,DAGISEL %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck --check-prefixes=CHECK,GISEL %s

; The nonnull flag on addrspacecast allows the target to skip the null check
; that maps a source null pointer to the destination null pointer.

define void @local_to_flat(ptr addrspace(3) %ptr) {
; CHECK-LABEL: local_to_flat:
; CHECK:       ; %bb.0:
; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT:    s_mov_b64 s[4:5], src_shared_base
; CHECK-NEXT:    v_mov_b32_e32 v1, s5
; CHECK-NEXT:    v_mov_b32_e32 v2, 7
; CHECK-NEXT:    flat_store_dword v[0:1], v2
; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT:    s_setpc_b64 s[30:31]
  %1 = addrspacecast nonnull ptr addrspace(3) %ptr to ptr
  store volatile i32 7, ptr %1, align 4
  ret void
}

define void @private_to_flat(ptr addrspace(5) %ptr) {
; CHECK-LABEL: private_to_flat:
; CHECK:       ; %bb.0:
; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT:    s_mov_b64 s[4:5], src_private_base
; CHECK-NEXT:    v_mov_b32_e32 v1, s5
; CHECK-NEXT:    v_mov_b32_e32 v2, 7
; CHECK-NEXT:    flat_store_dword v[0:1], v2
; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT:    s_setpc_b64 s[30:31]
  %1 = addrspacecast nonnull ptr addrspace(5) %ptr to ptr
  store volatile i32 7, ptr %1, align 4
  ret void
}

define void @flat_to_local(ptr %ptr) {
; CHECK-LABEL: flat_to_local:
; CHECK:       ; %bb.0:
; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT:    v_mov_b32_e32 v1, 7
; CHECK-NEXT:    ds_write_b32 v0, v1
; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
; CHECK-NEXT:    s_setpc_b64 s[30:31]
  %1 = addrspacecast nonnull ptr %ptr to ptr addrspace(3)
  store volatile i32 7, ptr addrspace(3) %1, align 4
  ret void
}

define void @flat_to_private(ptr %ptr) {
; CHECK-LABEL: flat_to_private:
; CHECK:       ; %bb.0:
; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT:    v_mov_b32_e32 v1, 7
; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
; CHECK-NEXT:    s_waitcnt vmcnt(0)
; CHECK-NEXT:    s_setpc_b64 s[30:31]
  %1 = addrspacecast nonnull ptr %ptr to ptr addrspace(5)
  store volatile i32 7, ptr addrspace(5) %1, align 4
  ret void
}

; The nonnull flag must survive scalarization of a vector addrspacecast, so no
; per-element null check is emitted.
define <2 x ptr> @local_to_flat_vector(<2 x ptr addrspace(3)> %ptr) {
; CHECK-LABEL: local_to_flat_vector:
; CHECK:       ; %bb.0:
; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT:    s_mov_b64 s[4:5], src_shared_base
; CHECK-NEXT:    v_mov_b32_e32 v2, v1
; CHECK-NEXT:    v_mov_b32_e32 v1, s5
; CHECK-NEXT:    v_mov_b32_e32 v3, s5
; CHECK-NEXT:    s_setpc_b64 s[30:31]
  %1 = addrspacecast nonnull <2 x ptr addrspace(3)> %ptr to <2 x ptr>
  ret <2 x ptr> %1
}

; The nonnull flag must also survive vector type-legalization that splits the
; addrspacecast (v32 splits repeatedly).
define <32 x ptr> @local_to_flat_vector_split(<32 x ptr addrspace(3)> %ptr) {
; DAGISEL-LABEL: local_to_flat_vector_split:
; DAGISEL:       ; %bb.0:
; DAGISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; DAGISEL-NEXT:    buffer_store_dword v30, v0, s[0:3], 0 offen offset:232
; DAGISEL-NEXT:    buffer_store_dword v29, v0, s[0:3], 0 offen offset:224
; DAGISEL-NEXT:    buffer_store_dword v28, v0, s[0:3], 0 offen offset:216
; DAGISEL-NEXT:    buffer_store_dword v27, v0, s[0:3], 0 offen offset:208
; DAGISEL-NEXT:    buffer_store_dword v26, v0, s[0:3], 0 offen offset:200
; DAGISEL-NEXT:    buffer_load_dword v26, off, s[0:3], s32 offset:4
; DAGISEL-NEXT:    s_nop 0
; DAGISEL-NEXT:    buffer_load_dword v27, off, s[0:3], s32
; DAGISEL-NEXT:    s_mov_b64 s[4:5], src_shared_base
; DAGISEL-NEXT:    buffer_store_dword v25, v0, s[0:3], 0 offen offset:192
; DAGISEL-NEXT:    buffer_store_dword v24, v0, s[0:3], 0 offen offset:184
; DAGISEL-NEXT:    buffer_store_dword v23, v0, s[0:3], 0 offen offset:176
; DAGISEL-NEXT:    buffer_store_dword v22, v0, s[0:3], 0 offen offset:168
; DAGISEL-NEXT:    buffer_store_dword v21, v0, s[0:3], 0 offen offset:160
; DAGISEL-NEXT:    buffer_store_dword v20, v0, s[0:3], 0 offen offset:152
; DAGISEL-NEXT:    buffer_store_dword v19, v0, s[0:3], 0 offen offset:144
; DAGISEL-NEXT:    buffer_store_dword v18, v0, s[0:3], 0 offen offset:136
; DAGISEL-NEXT:    buffer_store_dword v17, v0, s[0:3], 0 offen offset:128
; DAGISEL-NEXT:    buffer_store_dword v16, v0, s[0:3], 0 offen offset:120
; DAGISEL-NEXT:    buffer_store_dword v15, v0, s[0:3], 0 offen offset:112
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:104
; DAGISEL-NEXT:    v_mov_b32_e32 v14, s5
; DAGISEL-NEXT:    buffer_store_dword v13, v0, s[0:3], 0 offen offset:96
; DAGISEL-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen offset:88
; DAGISEL-NEXT:    buffer_store_dword v11, v0, s[0:3], 0 offen offset:80
; DAGISEL-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:72
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:252
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:244
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:236
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:228
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:220
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:212
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:204
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:196
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:188
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:180
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:172
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:164
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:156
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:148
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:140
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:132
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:124
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:116
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:108
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:100
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:92
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:84
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:76
; DAGISEL-NEXT:    s_waitcnt vmcnt(40)
; DAGISEL-NEXT:    buffer_store_dword v26, v0, s[0:3], 0 offen offset:248
; DAGISEL-NEXT:    s_waitcnt vmcnt(40)
; DAGISEL-NEXT:    buffer_store_dword v27, v0, s[0:3], 0 offen offset:240
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:68
; DAGISEL-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:64
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:60
; DAGISEL-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:56
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:52
; DAGISEL-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:48
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:44
; DAGISEL-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:40
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:36
; DAGISEL-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:32
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:28
; DAGISEL-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:24
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:20
; DAGISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:16
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:12
; DAGISEL-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:8
; DAGISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:4
; DAGISEL-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
; DAGISEL-NEXT:    s_waitcnt vmcnt(0)
; DAGISEL-NEXT:    s_setpc_b64 s[30:31]
;
; GISEL-LABEL: local_to_flat_vector_split:
; GISEL:       ; %bb.0:
; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
; GISEL-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:8
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:16
; GISEL-NEXT:    buffer_store_dword v4, v0, s[0:3], 0 offen offset:24
; GISEL-NEXT:    buffer_load_dword v1, off, s[0:3], s32
; GISEL-NEXT:    s_mov_b64 s[4:5], src_shared_base
; GISEL-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:4
; GISEL-NEXT:    v_mov_b32_e32 v3, s5
; GISEL-NEXT:    buffer_store_dword v5, v0, s[0:3], 0 offen offset:32
; GISEL-NEXT:    buffer_store_dword v6, v0, s[0:3], 0 offen offset:40
; GISEL-NEXT:    buffer_store_dword v7, v0, s[0:3], 0 offen offset:48
; GISEL-NEXT:    buffer_store_dword v8, v0, s[0:3], 0 offen offset:56
; GISEL-NEXT:    buffer_store_dword v9, v0, s[0:3], 0 offen offset:64
; GISEL-NEXT:    buffer_store_dword v10, v0, s[0:3], 0 offen offset:72
; GISEL-NEXT:    buffer_store_dword v11, v0, s[0:3], 0 offen offset:80
; GISEL-NEXT:    buffer_store_dword v12, v0, s[0:3], 0 offen offset:88
; GISEL-NEXT:    buffer_store_dword v13, v0, s[0:3], 0 offen offset:96
; GISEL-NEXT:    buffer_store_dword v14, v0, s[0:3], 0 offen offset:104
; GISEL-NEXT:    buffer_store_dword v15, v0, s[0:3], 0 offen offset:112
; GISEL-NEXT:    buffer_store_dword v16, v0, s[0:3], 0 offen offset:120
; GISEL-NEXT:    buffer_store_dword v17, v0, s[0:3], 0 offen offset:128
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:4
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:12
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:20
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:28
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:36
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:44
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:52
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:60
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:68
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:76
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:84
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:92
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:100
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:108
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:116
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:124
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:132
; GISEL-NEXT:    buffer_store_dword v18, v0, s[0:3], 0 offen offset:136
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:140
; GISEL-NEXT:    buffer_store_dword v19, v0, s[0:3], 0 offen offset:144
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:148
; GISEL-NEXT:    buffer_store_dword v20, v0, s[0:3], 0 offen offset:152
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:156
; GISEL-NEXT:    buffer_store_dword v21, v0, s[0:3], 0 offen offset:160
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:164
; GISEL-NEXT:    buffer_store_dword v22, v0, s[0:3], 0 offen offset:168
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:172
; GISEL-NEXT:    buffer_store_dword v23, v0, s[0:3], 0 offen offset:176
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:180
; GISEL-NEXT:    buffer_store_dword v24, v0, s[0:3], 0 offen offset:184
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:188
; GISEL-NEXT:    buffer_store_dword v25, v0, s[0:3], 0 offen offset:192
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:196
; GISEL-NEXT:    buffer_store_dword v26, v0, s[0:3], 0 offen offset:200
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:204
; GISEL-NEXT:    buffer_store_dword v27, v0, s[0:3], 0 offen offset:208
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:212
; GISEL-NEXT:    buffer_store_dword v28, v0, s[0:3], 0 offen offset:216
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:220
; GISEL-NEXT:    buffer_store_dword v29, v0, s[0:3], 0 offen offset:224
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:228
; GISEL-NEXT:    buffer_store_dword v30, v0, s[0:3], 0 offen offset:232
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:236
; GISEL-NEXT:    s_waitcnt vmcnt(57)
; GISEL-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen offset:240
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:244
; GISEL-NEXT:    s_waitcnt vmcnt(58)
; GISEL-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen offset:248
; GISEL-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen offset:252
; GISEL-NEXT:    s_waitcnt vmcnt(0)
; GISEL-NEXT:    s_setpc_b64 s[30:31]
  %1 = addrspacecast nonnull <32 x ptr addrspace(3)> %ptr to <32 x ptr>
  ret <32 x ptr> %1
}

; Two nonnull casts of the same flat source to different destination address
; spaces must not be CSEd together despite sharing an operand and the flag.
define void @nonnull_casts_different_dest(i64 %i) {
; CHECK-LABEL: nonnull_casts_different_dest:
; CHECK:       ; %bb.0:
; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT:    v_mov_b32_e32 v1, 7
; CHECK-NEXT:    ds_write_b32 v0, v1
; CHECK-NEXT:    v_mov_b32_e32 v1, 8
; CHECK-NEXT:    buffer_store_dword v1, v0, s[0:3], 0 offen
; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT:    s_setpc_b64 s[30:31]
  %flat = inttoptr i64 %i to ptr
  %local = addrspacecast nonnull ptr %flat to ptr addrspace(3)
  %priv = addrspacecast nonnull ptr %flat to ptr addrspace(5)
  store volatile i32 7, ptr addrspace(3) %local, align 4
  store volatile i32 8, ptr addrspace(5) %priv, align 4
  ret void
}

; The inverted case: nonnull casts to flat from two different source address
; spaces. Each source has its own null value, so the casts are distinct.
define void @nonnull_casts_different_src(i32 %i) {
; DAGISEL-LABEL: nonnull_casts_different_src:
; DAGISEL:       ; %bb.0:
; DAGISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; DAGISEL-NEXT:    s_mov_b64 s[4:5], src_shared_base
; DAGISEL-NEXT:    s_mov_b64 s[6:7], src_private_base
; DAGISEL-NEXT:    v_mov_b32_e32 v1, s5
; DAGISEL-NEXT:    v_mov_b32_e32 v4, 7
; DAGISEL-NEXT:    v_mov_b32_e32 v2, v0
; DAGISEL-NEXT:    v_mov_b32_e32 v3, s7
; DAGISEL-NEXT:    flat_store_dword v[0:1], v4
; DAGISEL-NEXT:    s_waitcnt vmcnt(0)
; DAGISEL-NEXT:    v_mov_b32_e32 v0, 8
; DAGISEL-NEXT:    flat_store_dword v[2:3], v0
; DAGISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; DAGISEL-NEXT:    s_setpc_b64 s[30:31]
;
; GISEL-LABEL: nonnull_casts_different_src:
; GISEL:       ; %bb.0:
; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT:    s_mov_b64 s[4:5], src_shared_base
; GISEL-NEXT:    s_mov_b64 s[6:7], src_private_base
; GISEL-NEXT:    v_mov_b32_e32 v1, s5
; GISEL-NEXT:    v_mov_b32_e32 v4, 7
; GISEL-NEXT:    v_mov_b32_e32 v3, s7
; GISEL-NEXT:    v_mov_b32_e32 v2, v0
; GISEL-NEXT:    flat_store_dword v[0:1], v4
; GISEL-NEXT:    s_waitcnt vmcnt(0)
; GISEL-NEXT:    v_mov_b32_e32 v0, 8
; GISEL-NEXT:    flat_store_dword v[2:3], v0
; GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
; GISEL-NEXT:    s_setpc_b64 s[30:31]
  %local = inttoptr i32 %i to ptr addrspace(3)
  %priv = inttoptr i32 %i to ptr addrspace(5)
  %flat.local = addrspacecast nonnull ptr addrspace(3) %local to ptr
  %flat.priv = addrspacecast nonnull ptr addrspace(5) %priv to ptr
  store volatile i32 7, ptr %flat.local, align 4
  store volatile i32 8, ptr %flat.priv, align 4
  ret void
}