; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -S -mtriple=amdgpu9.00-amd-amdhsa -amdgpu-late-codegenprepare %s | FileCheck %s -check-prefix=GFX9
; RUN: opt -S -mtriple=amdgpu12.00-amd-amdhsa -amdgpu-late-codegenprepare %s | FileCheck %s -check-prefix=GFX12
; RUN: opt -S -mtriple=amdgpu9.00-amd-amdhsa -passes=amdgpu-late-codegenprepare %s | FileCheck %s -check-prefix=GFX9

; Make sure we don't crash when trying to create a bitcast between
; address spaces
define amdgpu_kernel void @constant_from_offset_cast_generic_null() {
; GFX9-LABEL: @constant_from_offset_cast_generic_null(
; GFX9-NEXT:    [[TMP3:%.*]] = load i8, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr null to ptr addrspace(4)), i64 6), align 1
; GFX9-NEXT:    store i8 [[TMP3]], ptr addrspace(1) poison, align 1
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @constant_from_offset_cast_generic_null(
; GFX12-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr null to ptr addrspace(4)), i64 6), align 1
; GFX12-NEXT:    store i8 [[LOAD]], ptr addrspace(1) poison, align 1
; GFX12-NEXT:    ret void
;
  %load = load i8, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr null to ptr addrspace(4)), i64 6), align 1
  store i8 %load, ptr addrspace(1) poison
  ret void
}

define amdgpu_kernel void @constant_from_offset_cast_global_null() {
; GFX9-LABEL: @constant_from_offset_cast_global_null(
; GFX9-NEXT:    [[TMP3:%.*]] = load i8, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr addrspace(1) null to ptr addrspace(4)), i64 6), align 1
; GFX9-NEXT:    store i8 [[TMP3]], ptr addrspace(1) poison, align 1
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @constant_from_offset_cast_global_null(
; GFX12-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr addrspace(1) null to ptr addrspace(4)), i64 6), align 1
; GFX12-NEXT:    store i8 [[LOAD]], ptr addrspace(1) poison, align 1
; GFX12-NEXT:    ret void
;
  %load = load i8, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr addrspace(1) null to ptr addrspace(4)), i64 6), align 1
  store i8 %load, ptr addrspace(1) poison
  ret void
}

@gv = unnamed_addr addrspace(1) global [64 x i8] poison, align 4

define amdgpu_kernel void @constant_from_offset_cast_global_gv() {
; GFX9-LABEL: @constant_from_offset_cast_global_gv(
; GFX9-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) getelementptr (i8, ptr addrspace(4) addrspacecast (ptr addrspace(1) @gv to ptr addrspace(4)), i64 4), align 4
; GFX9-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 16
; GFX9-NEXT:    [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
; GFX9-NEXT:    store i8 [[TMP3]], ptr addrspace(1) poison, align 1
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @constant_from_offset_cast_global_gv(
; GFX12-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr addrspace(1) @gv to ptr addrspace(4)), i64 6), align 1
; GFX12-NEXT:    store i8 [[LOAD]], ptr addrspace(1) poison, align 1
; GFX12-NEXT:    ret void
;
  %load = load i8, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr addrspace(1) @gv to ptr addrspace(4)), i64 6), align 1
  store i8 %load, ptr addrspace(1) poison
  ret void
}

define amdgpu_kernel void @constant_from_offset_cast_generic_inttoptr() {
; GFX9-LABEL: @constant_from_offset_cast_generic_inttoptr(
; GFX9-NEXT:    [[TMP3:%.*]] = load i8, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr inttoptr (i64 128 to ptr) to ptr addrspace(4)), i64 6), align 1
; GFX9-NEXT:    store i8 [[TMP3]], ptr addrspace(1) poison, align 1
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @constant_from_offset_cast_generic_inttoptr(
; GFX12-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr inttoptr (i64 128 to ptr) to ptr addrspace(4)), i64 6), align 1
; GFX12-NEXT:    store i8 [[LOAD]], ptr addrspace(1) poison, align 1
; GFX12-NEXT:    ret void
;
  %load = load i8, ptr addrspace(4) getelementptr inbounds (i8, ptr addrspace(4) addrspacecast (ptr inttoptr (i64 128 to ptr) to ptr addrspace(4)), i64 6), align 1
  store i8 %load, ptr addrspace(1) poison
  ret void
}

define amdgpu_kernel void @constant_from_inttoptr() {
; GFX9-LABEL: @constant_from_inttoptr(
; GFX9-NEXT:    [[TMP2:%.*]] = load i8, ptr addrspace(4) inttoptr (i64 128 to ptr addrspace(4)), align 1
; GFX9-NEXT:    store i8 [[TMP2]], ptr addrspace(1) poison, align 1
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @constant_from_inttoptr(
; GFX12-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(4) inttoptr (i64 128 to ptr addrspace(4)), align 1
; GFX12-NEXT:    store i8 [[LOAD]], ptr addrspace(1) poison, align 1
; GFX12-NEXT:    ret void
;
  %load = load i8, ptr addrspace(4) inttoptr (i64 128 to ptr addrspace(4)), align 1
  store i8 %load, ptr addrspace(1) poison
  ret void
}

define amdgpu_kernel void @constant_i16_from_dword_aligned() {
; GFX9-LABEL: @constant_i16_from_dword_aligned(
; GFX9-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(4) inttoptr (i64 128 to ptr addrspace(4)), align 2
; GFX9-NEXT:    store i16 [[TMP2]], ptr addrspace(1) poison, align 2
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @constant_i16_from_dword_aligned(
; GFX12-NEXT:    [[LOAD:%.*]] = load i16, ptr addrspace(4) inttoptr (i64 128 to ptr addrspace(4)), align 2
; GFX12-NEXT:    store i16 [[LOAD]], ptr addrspace(1) poison, align 2
; GFX12-NEXT:    ret void
;
  %load = load i16, ptr addrspace(4) inttoptr (i64 128 to ptr addrspace(4)), align 2
  store i16 %load, ptr addrspace(1) poison
  ret void
}

define i1 @constant_i16_from_null() {
; GFX9-LABEL: @constant_i16_from_null(
; GFX9-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(4) null, align 2
; GFX9-NEXT:    [[CMP:%.*]] = icmp ne i16 [[TMP2]], 0
; GFX9-NEXT:    ret i1 [[CMP]]
;
; GFX12-LABEL: @constant_i16_from_null(
; GFX12-NEXT:    [[LOAD:%.*]] = load i16, ptr addrspace(4) null, align 2
; GFX12-NEXT:    [[CMP:%.*]] = icmp ne i16 [[LOAD]], 0
; GFX12-NEXT:    ret i1 [[CMP]]
;
  %load = load i16, ptr addrspace(4) null, align 2
  %cmp = icmp ne i16 %load, 0
  ret i1 %cmp
}

define void @broken_phi() {
; GFX9-LABEL: @broken_phi(
; GFX9-NEXT:  bb:
; GFX9-NEXT:    br label [[BB1:%.*]]
; GFX9:       bb1:
; GFX9-NEXT:    [[I:%.*]] = phi <4 x i8> [ splat (i8 1), [[BB:%.*]] ], [ [[I8:%.*]], [[BB7:%.*]] ]
; GFX9-NEXT:    br i1 false, label [[BB3:%.*]], label [[BB2:%.*]]
; GFX9:       bb2:
; GFX9-NEXT:    br label [[BB3]]
; GFX9:       bb3:
; GFX9-NEXT:    [[I4:%.*]] = phi <4 x i8> [ zeroinitializer, [[BB2]] ], [ [[I]], [[BB1]] ]
; GFX9-NEXT:    br i1 false, label [[BB7]], label [[BB5:%.*]]
; GFX9:       bb5:
; GFX9-NEXT:    [[I6:%.*]] = call <4 x i8> @llvm.smax.v4i8(<4 x i8> [[I4]], <4 x i8> zeroinitializer)
; GFX9-NEXT:    br label [[BB7]]
; GFX9:       bb7:
; GFX9-NEXT:    [[I8]] = phi <4 x i8> [ zeroinitializer, [[BB5]] ], [ zeroinitializer, [[BB3]] ]
; GFX9-NEXT:    br label [[BB1]]
;
; GFX12-LABEL: @broken_phi(
; GFX12-NEXT:  bb:
; GFX12-NEXT:    br label [[BB1:%.*]]
; GFX12:       bb1:
; GFX12-NEXT:    [[I:%.*]] = phi <4 x i8> [ splat (i8 1), [[BB:%.*]] ], [ [[I8:%.*]], [[BB7:%.*]] ]
; GFX12-NEXT:    br i1 false, label [[BB3:%.*]], label [[BB2:%.*]]
; GFX12:       bb2:
; GFX12-NEXT:    br label [[BB3]]
; GFX12:       bb3:
; GFX12-NEXT:    [[I4:%.*]] = phi <4 x i8> [ zeroinitializer, [[BB2]] ], [ [[I]], [[BB1]] ]
; GFX12-NEXT:    br i1 false, label [[BB7]], label [[BB5:%.*]]
; GFX12:       bb5:
; GFX12-NEXT:    [[I6:%.*]] = call <4 x i8> @llvm.smax.v4i8(<4 x i8> [[I4]], <4 x i8> zeroinitializer)
; GFX12-NEXT:    br label [[BB7]]
; GFX12:       bb7:
; GFX12-NEXT:    [[I8]] = phi <4 x i8> [ zeroinitializer, [[BB5]] ], [ zeroinitializer, [[BB3]] ]
; GFX12-NEXT:    br label [[BB1]]
;
bb:
  br label %bb1
bb1:
  %i = phi <4 x i8> [ <i8 1, i8 1, i8 1, i8 1>, %bb ], [ %i8, %bb7 ]
  br i1 false, label %bb3, label %bb2
bb2:
  br label %bb3
bb3:
  %i4 = phi <4 x i8> [ zeroinitializer, %bb2 ], [ %i, %bb1 ]
  br i1 false, label %bb7, label %bb5
bb5:
  %i6 = call <4 x i8> @llvm.smax.v4i8(<4 x i8> %i4, <4 x i8> zeroinitializer)
  br label %bb7
bb7:
  %i8 = phi <4 x i8> [ zeroinitializer, %bb5 ], [ zeroinitializer, %bb3 ]
  br label %bb1
}

; The widened i32 load reads bytes outside the original i8 load, so !noundef
; must not be carried over for GFX9.
define amdgpu_kernel void @no_widen_noundef(ptr addrspace(4) align 4 %p, ptr addrspace(1) %out) {
; GFX9-LABEL: @no_widen_noundef(
; GFX9-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[P:%.*]], i64 1
; GFX9-NEXT:    [[TMP4:%.*]] = load i8, ptr addrspace(4) [[P1]], align 1, !noundef [[META0:![0-9]+]]
; GFX9-NEXT:    [[VZ:%.*]] = zext i8 [[TMP4]] to i32
; GFX9-NEXT:    store i32 [[VZ]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @no_widen_noundef(
; GFX12-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[P:%.*]], i64 1
; GFX12-NEXT:    [[V:%.*]] = load i8, ptr addrspace(4) [[P1]], align 1, !noundef [[META0:![0-9]+]]
; GFX12-NEXT:    [[VZ:%.*]] = zext i8 [[V]] to i32
; GFX12-NEXT:    store i32 [[VZ]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX12-NEXT:    ret void
;
  %p1 = getelementptr inbounds i8, ptr addrspace(4) %p, i64 1
  %v = load i8, ptr addrspace(4) %p1, align 1, !noundef !0
  %vz = zext i8 %v to i32
  store i32 %vz, ptr addrspace(1) %out, align 4
  ret void
}

; The widened i32 load reads bytes outside the original f16 load, so !nofpclass
; must not be carried over for GFX9.
define amdgpu_kernel void @no_widen_nofpclass(ptr addrspace(4) align 4 %p, ptr addrspace(1) %out) {
; GFX9-LABEL: @no_widen_nofpclass(
; GFX9-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[P:%.*]], i64 2
; GFX9-NEXT:    [[TMP5:%.*]] = load half, ptr addrspace(4) [[P1]], align 2, !nofpclass [[META1:![0-9]+]]
; GFX9-NEXT:    [[VB:%.*]] = bitcast half [[TMP5]] to i16
; GFX9-NEXT:    [[VZ:%.*]] = zext i16 [[VB]] to i32
; GFX9-NEXT:    store i32 [[VZ]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @no_widen_nofpclass(
; GFX12-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[P:%.*]], i64 2
; GFX12-NEXT:    [[V:%.*]] = load half, ptr addrspace(4) [[P1]], align 2, !nofpclass [[META1:![0-9]+]]
; GFX12-NEXT:    [[VB:%.*]] = bitcast half [[V]] to i16
; GFX12-NEXT:    [[VZ:%.*]] = zext i16 [[VB]] to i32
; GFX12-NEXT:    store i32 [[VZ]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX12-NEXT:    ret void
;
  %p1 = getelementptr inbounds i8, ptr addrspace(4) %p, i64 2
  %v = load half, ptr addrspace(4) %p1, align 2, !nofpclass !1
  %vb = bitcast half %v to i16
  %vz = zext i16 %vb to i32
  store i32 %vz, ptr addrspace(1) %out, align 4
  ret void
}

; dereferenceable(4) covers the padding bytes of the widened access, so widening is safe.
define amdgpu_kernel void @widen_dereferenceable_dword(ptr addrspace(4) align 4 dereferenceable(4) %p, ptr addrspace(1) %out) {
; GFX9-LABEL: @widen_dereferenceable_dword(
; GFX9-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[P:%.*]], i64 0
; GFX9-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4
; GFX9-NEXT:    [[TMP3:%.*]] = trunc i32 [[TMP2]] to i16
; GFX9-NEXT:    [[VZ:%.*]] = zext i16 [[TMP3]] to i32
; GFX9-NEXT:    store i32 [[VZ]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @widen_dereferenceable_dword(
; GFX12-NEXT:    [[V:%.*]] = load i16, ptr addrspace(4) [[P:%.*]], align 2
; GFX12-NEXT:    [[VZ:%.*]] = zext i16 [[V]] to i32
; GFX12-NEXT:    store i32 [[VZ]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX12-NEXT:    ret void
;
  %v = load i16, ptr addrspace(4) %p, align 2
  %vz = zext i16 %v to i32
  store i32 %vz, ptr addrspace(1) %out, align 4
  ret void
}

; dereferenceable(2) does not cover the padding bytes of the widened access, so widening must not occur.
define amdgpu_kernel void @no_widen_insufficient_dereferenceable(ptr addrspace(4) align 4 dereferenceable(2) %p, ptr addrspace(1) %out) {
; GFX9-LABEL: @no_widen_insufficient_dereferenceable(
; GFX9-NEXT:    [[V:%.*]] = load i16, ptr addrspace(4) [[P:%.*]], align 2
; GFX9-NEXT:    [[VZ:%.*]] = zext i16 [[V]] to i32
; GFX9-NEXT:    store i32 [[VZ]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @no_widen_insufficient_dereferenceable(
; GFX12-NEXT:    [[V:%.*]] = load i16, ptr addrspace(4) [[P:%.*]], align 2
; GFX12-NEXT:    [[VZ:%.*]] = zext i16 [[V]] to i32
; GFX12-NEXT:    store i32 [[VZ]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX12-NEXT:    ret void
;
  %v = load i16, ptr addrspace(4) %p, align 2
  %vz = zext i16 %v to i32
  store i32 %vz, ptr addrspace(1) %out, align 4
  ret void
}

!0 = !{}
!1 = !{i32 3}

; Coercing a value-producing terminator's def used to insert past the block end.
define amdgpu_kernel void @callbr_def_cross_block_use(ptr addrspace(1) %out) {
;
; GFX9-LABEL: @callbr_def_cross_block_use(
; GFX9-NEXT:  entry:
; GFX9-NEXT:    [[V:%.*]] = callbr <4 x i8> asm "", "=v"()
; GFX9-NEXT:            to label [[USE:%.*]] []
; GFX9:       use:
; GFX9-NEXT:    store <4 x i8> [[V]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @callbr_def_cross_block_use(
; GFX12-NEXT:  entry:
; GFX12-NEXT:    [[V:%.*]] = callbr <4 x i8> asm "", "=v"()
; GFX12-NEXT:            to label [[USE:%.*]] []
; GFX12:       use:
; GFX12-NEXT:    store <4 x i8> [[V]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX12-NEXT:    ret void
;
entry:
  %v = callbr <4 x i8> asm "", "=v"() to label %use []
use:
  store <4 x i8> %v, ptr addrspace(1) %out
  ret void
}

; Coercible incoming value shares a phi with the callbr def; the new phi must unwind.
define amdgpu_kernel void @callbr_def_phi_incoming(ptr addrspace(1) %in, ptr addrspace(1) %out, i1 %c) {
;
; GFX9-LABEL: @callbr_def_phi_incoming(
; GFX9-NEXT:  entry:
; GFX9-NEXT:    br i1 [[C:%.*]], label [[A:%.*]], label [[B:%.*]]
; GFX9:       a:
; GFX9-NEXT:    [[V:%.*]] = callbr <4 x i8> asm "", "=v"()
; GFX9-NEXT:            to label [[JOIN:%.*]] []
; GFX9:       b:
; GFX9-NEXT:    [[W:%.*]] = load <4 x i8>, ptr addrspace(1) [[IN:%.*]], align 4
; GFX9-NEXT:    br label [[JOIN]]
; GFX9:       join:
; GFX9-NEXT:    [[P:%.*]] = phi <4 x i8> [ [[V]], [[A]] ], [ [[W]], [[B]] ]
; GFX9-NEXT:    store <4 x i8> [[P]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @callbr_def_phi_incoming(
; GFX12-NEXT:  entry:
; GFX12-NEXT:    br i1 [[C:%.*]], label [[A:%.*]], label [[B:%.*]]
; GFX12:       a:
; GFX12-NEXT:    [[V:%.*]] = callbr <4 x i8> asm "", "=v"()
; GFX12-NEXT:            to label [[JOIN:%.*]] []
; GFX12:       b:
; GFX12-NEXT:    [[W:%.*]] = load <4 x i8>, ptr addrspace(1) [[IN:%.*]], align 4
; GFX12-NEXT:    br label [[JOIN]]
; GFX12:       join:
; GFX12-NEXT:    [[P:%.*]] = phi <4 x i8> [ [[V]], [[A]] ], [ [[W]], [[B]] ]
; GFX12-NEXT:    store <4 x i8> [[P]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX12-NEXT:    ret void
;
entry:
  br i1 %c, label %a, label %b
a:
  %v = callbr <4 x i8> asm "", "=v"() to label %join []
b:
  %w = load <4 x i8>, ptr addrspace(1) %in
  br label %join
join:
  %p = phi <4 x i8> [ %v, %a ], [ %w, %b ]
  store <4 x i8> %p, ptr addrspace(1) %out
  ret void
}

; Loop-carried phi feeding itself: unwound replacement phis form a use cycle.
define amdgpu_kernel void @callbr_def_self_loop_phi(ptr addrspace(1) %out, i1 %c) {
;
; GFX9-LABEL: @callbr_def_self_loop_phi(
; GFX9-NEXT:  entry:
; GFX9-NEXT:    [[V:%.*]] = callbr <4 x i8> asm "", "=v"()
; GFX9-NEXT:            to label [[LOOP:%.*]] []
; GFX9:       loop:
; GFX9-NEXT:    [[P:%.*]] = phi <4 x i8> [ [[V]], [[ENTRY:%.*]] ], [ [[P]], [[LOOP]] ]
; GFX9-NEXT:    br i1 [[C:%.*]], label [[LOOP]], label [[EXIT:%.*]]
; GFX9:       exit:
; GFX9-NEXT:    store <4 x i8> [[P]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @callbr_def_self_loop_phi(
; GFX12-NEXT:  entry:
; GFX12-NEXT:    [[V:%.*]] = callbr <4 x i8> asm "", "=v"()
; GFX12-NEXT:            to label [[LOOP:%.*]] []
; GFX12:       loop:
; GFX12-NEXT:    [[P:%.*]] = phi <4 x i8> [ [[V]], [[ENTRY:%.*]] ], [ [[P]], [[LOOP]] ]
; GFX12-NEXT:    br i1 [[C:%.*]], label [[LOOP]], label [[EXIT:%.*]]
; GFX12:       exit:
; GFX12-NEXT:    store <4 x i8> [[P]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX12-NEXT:    ret void
;
entry:
  %v = callbr <4 x i8> asm "", "=v"() to label %loop []
loop:
  %p = phi <4 x i8> [ %v, %entry ], [ %p, %loop ]
  br i1 %c, label %loop, label %exit
exit:
  store <4 x i8> %p, ptr addrspace(1) %out
  ret void
}

; Used on an indirect edge, so there is no single block to insert the coercion at.
define amdgpu_kernel void @callbr_def_indirect_use(ptr addrspace(1) %out) {
;
; GFX9-LABEL: @callbr_def_indirect_use(
; GFX9-NEXT:  entry:
; GFX9-NEXT:    [[V:%.*]] = callbr <4 x i8> asm "", "=v,!i"()
; GFX9-NEXT:            to label [[FALLTHROUGH:%.*]] [label [[INDIRECT:%.*]]]
; GFX9:       fallthrough:
; GFX9-NEXT:    store <4 x i8> zeroinitializer, ptr addrspace(1) [[OUT:%.*]], align 4
; GFX9-NEXT:    ret void
; GFX9:       indirect:
; GFX9-NEXT:    store <4 x i8> [[V]], ptr addrspace(1) [[OUT]], align 4
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @callbr_def_indirect_use(
; GFX12-NEXT:  entry:
; GFX12-NEXT:    [[V:%.*]] = callbr <4 x i8> asm "", "=v,!i"()
; GFX12-NEXT:            to label [[FALLTHROUGH:%.*]] [label [[INDIRECT:%.*]]]
; GFX12:       fallthrough:
; GFX12-NEXT:    store <4 x i8> zeroinitializer, ptr addrspace(1) [[OUT:%.*]], align 4
; GFX12-NEXT:    ret void
; GFX12:       indirect:
; GFX12-NEXT:    store <4 x i8> [[V]], ptr addrspace(1) [[OUT]], align 4
; GFX12-NEXT:    ret void
;
entry:
  %v = callbr <4 x i8> asm "", "=v,!i"() to label %fallthrough [label %indirect]
fallthrough:
  store <4 x i8> zeroinitializer, ptr addrspace(1) %out
  ret void
indirect:
  store <4 x i8> %v, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @invoke_def_cross_block_use(ptr addrspace(1) %out) personality ptr @__gxx_personality_v0 {
;
; GFX9-LABEL: @invoke_def_cross_block_use(
; GFX9-NEXT:  entry:
; GFX9-NEXT:    [[V:%.*]] = invoke <4 x i8> @ret_v4i8()
; GFX9-NEXT:            to label [[USE:%.*]] unwind label [[LPAD:%.*]]
; GFX9:       use:
; GFX9-NEXT:    store <4 x i8> [[V]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX9-NEXT:    ret void
; GFX9:       lpad:
; GFX9-NEXT:    [[LP:%.*]] = landingpad { ptr, i32 }
; GFX9-NEXT:            cleanup
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @invoke_def_cross_block_use(
; GFX12-NEXT:  entry:
; GFX12-NEXT:    [[V:%.*]] = invoke <4 x i8> @ret_v4i8()
; GFX12-NEXT:            to label [[USE:%.*]] unwind label [[LPAD:%.*]]
; GFX12:       use:
; GFX12-NEXT:    store <4 x i8> [[V]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX12-NEXT:    ret void
; GFX12:       lpad:
; GFX12-NEXT:    [[LP:%.*]] = landingpad { ptr, i32 }
; GFX12-NEXT:            cleanup
; GFX12-NEXT:    ret void
;
entry:
  %v = invoke <4 x i8> @ret_v4i8() to label %use unwind label %lpad
use:
  store <4 x i8> %v, ptr addrspace(1) %out
  ret void
lpad:
  %lp = landingpad { ptr, i32 } cleanup
  ret void
}

; Conversion must insert after the landingpad, not before it.
define amdgpu_kernel void @use_in_landingpad_block(ptr addrspace(1) %in, ptr addrspace(1) %out) personality ptr @__gxx_personality_v0 {
;
; GFX9-LABEL: @use_in_landingpad_block(
; GFX9-NEXT:  entry:
; GFX9-NEXT:    [[V:%.*]] = load <4 x i8>, ptr addrspace(1) [[IN:%.*]], align 4
; GFX9-NEXT:    [[V_BC:%.*]] = bitcast <4 x i8> [[V]] to i32
; GFX9-NEXT:    invoke void @maybe_throw()
; GFX9-NEXT:            to label [[CONT:%.*]] unwind label [[LPAD:%.*]]
; GFX9:       cont:
; GFX9-NEXT:    [[V_BC_BC1:%.*]] = bitcast i32 [[V_BC]] to <4 x i8>
; GFX9-NEXT:    store <4 x i8> [[V_BC_BC1]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX9-NEXT:    ret void
; GFX9:       lpad:
; GFX9-NEXT:    [[LP:%.*]] = landingpad { ptr, i32 }
; GFX9-NEXT:            cleanup
; GFX9-NEXT:    [[V_BC_BC:%.*]] = bitcast i32 [[V_BC]] to <4 x i8>
; GFX9-NEXT:    store <4 x i8> [[V_BC_BC]], ptr addrspace(1) [[OUT]], align 4
; GFX9-NEXT:    ret void
;
; GFX12-LABEL: @use_in_landingpad_block(
; GFX12-NEXT:  entry:
; GFX12-NEXT:    [[V:%.*]] = load <4 x i8>, ptr addrspace(1) [[IN:%.*]], align 4
; GFX12-NEXT:    [[V_BC:%.*]] = bitcast <4 x i8> [[V]] to i32
; GFX12-NEXT:    invoke void @maybe_throw()
; GFX12-NEXT:            to label [[CONT:%.*]] unwind label [[LPAD:%.*]]
; GFX12:       cont:
; GFX12-NEXT:    [[V_BC_BC1:%.*]] = bitcast i32 [[V_BC]] to <4 x i8>
; GFX12-NEXT:    store <4 x i8> [[V_BC_BC1]], ptr addrspace(1) [[OUT:%.*]], align 4
; GFX12-NEXT:    ret void
; GFX12:       lpad:
; GFX12-NEXT:    [[LP:%.*]] = landingpad { ptr, i32 }
; GFX12-NEXT:            cleanup
; GFX12-NEXT:    [[V_BC_BC:%.*]] = bitcast i32 [[V_BC]] to <4 x i8>
; GFX12-NEXT:    store <4 x i8> [[V_BC_BC]], ptr addrspace(1) [[OUT]], align 4
; GFX12-NEXT:    ret void
;
entry:
  %v = load <4 x i8>, ptr addrspace(1) %in
  invoke void @maybe_throw() to label %cont unwind label %lpad
cont:
  store <4 x i8> %v, ptr addrspace(1) %out
  ret void
lpad:
  %lp = landingpad { ptr, i32 } cleanup
  store <4 x i8> %v, ptr addrspace(1) %out
  ret void
}

declare <4 x i8> @ret_v4i8()
declare void @maybe_throw()
declare i32 @__gxx_personality_v0(...)